2024-11-27 01:24:14,772 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-27 01:24:14,788 main DEBUG Took 0.012721 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 01:24:14,788 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 01:24:14,789 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 01:24:14,790 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 01:24:14,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,798 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 01:24:14,815 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,818 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,819 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,820 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,821 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,823 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,823 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,824 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,825 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,826 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,826 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,827 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:24:14,828 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,829 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 01:24:14,831 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:24:14,832 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 01:24:14,835 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 01:24:14,835 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 01:24:14,837 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 01:24:14,837 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 01:24:14,847 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 01:24:14,850 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 01:24:14,852 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 01:24:14,852 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 01:24:14,853 main DEBUG createAppenders(={Console}) 2024-11-27 01:24:14,853 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-27 01:24:14,854 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-27 01:24:14,854 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-27 01:24:14,855 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 01:24:14,855 main DEBUG OutputStream closed 2024-11-27 01:24:14,856 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 01:24:14,856 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 01:24:14,856 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-27 01:24:14,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 01:24:14,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 01:24:14,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 01:24:14,950 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 01:24:14,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 01:24:14,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 01:24:14,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 01:24:14,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 01:24:14,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 01:24:14,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 01:24:14,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 01:24:14,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 01:24:14,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 01:24:14,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 01:24:14,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 01:24:14,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 01:24:14,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 01:24:14,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 01:24:14,959 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 01:24:14,959 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-27 01:24:14,959 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 01:24:14,960 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-27T01:24:15,284 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d 2024-11-27 01:24:15,287 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 01:24:15,288 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T01:24:15,298 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-27T01:24:15,305 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-27T01:24:15,338 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T01:24:15,407 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T01:24:15,407 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T01:24:15,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T01:24:15,440 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd, deleteOnExit=true 2024-11-27T01:24:15,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T01:24:15,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/test.cache.data in system properties and HBase conf 2024-11-27T01:24:15,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T01:24:15,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir in system properties and HBase conf 2024-11-27T01:24:15,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T01:24:15,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T01:24:15,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T01:24:15,552 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T01:24:15,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T01:24:15,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:24:15,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:24:15,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T01:24:15,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:24:15,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T01:24:15,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T01:24:15,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:24:15,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:24:15,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T01:24:15,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/nfs.dump.dir in system properties and HBase conf 2024-11-27T01:24:15,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/java.io.tmpdir in system properties and HBase conf 2024-11-27T01:24:15,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:24:15,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T01:24:15,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T01:24:16,690 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T01:24:16,787 INFO [Time-limited test {}] log.Log(170): Logging initialized @3066ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T01:24:16,892 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:24:16,981 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:24:17,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:24:17,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:24:17,021 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:24:17,035 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:24:17,037 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@139bf1d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:24:17,038 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1257581c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:24:17,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@126092e9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/java.io.tmpdir/jetty-localhost-40563-hadoop-hdfs-3_4_1-tests_jar-_-any-17323745685543178890/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:24:17,318 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cf0cac{HTTP/1.1, (http/1.1)}{localhost:40563} 2024-11-27T01:24:17,318 INFO [Time-limited test {}] server.Server(415): Started @3597ms 2024-11-27T01:24:17,830 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:24:17,839 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:24:17,841 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:24:17,841 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:24:17,842 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:24:17,842 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c12e3d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:24:17,843 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fecd6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:24:17,965 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d91a9f0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/java.io.tmpdir/jetty-localhost-42939-hadoop-hdfs-3_4_1-tests_jar-_-any-3691435881102734441/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:24:17,966 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2ace2437{HTTP/1.1, (http/1.1)}{localhost:42939} 2024-11-27T01:24:17,967 INFO [Time-limited test {}] server.Server(415): Started @4245ms 2024-11-27T01:24:18,023 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:24:18,151 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:24:18,156 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:24:18,159 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:24:18,160 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:24:18,160 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:24:18,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4477f2ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:24:18,162 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7deb8c77{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:24:18,295 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@80bbe1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/java.io.tmpdir/jetty-localhost-43955-hadoop-hdfs-3_4_1-tests_jar-_-any-12745936993786273614/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:24:18,296 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14fd625f{HTTP/1.1, (http/1.1)}{localhost:43955} 2024-11-27T01:24:18,296 INFO [Time-limited test {}] server.Server(415): Started @4575ms 2024-11-27T01:24:18,299 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:24:18,361 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:24:18,366 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:24:18,369 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:24:18,369 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:24:18,370 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:24:18,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32dd0934{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:24:18,372 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f216234{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:24:18,521 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data4/current/BP-272052841-172.17.0.2-1732670656425/current, will proceed with Du for space computation calculation, 2024-11-27T01:24:18,521 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data1/current/BP-272052841-172.17.0.2-1732670656425/current, will proceed with Du for space computation calculation, 2024-11-27T01:24:18,521 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data2/current/BP-272052841-172.17.0.2-1732670656425/current, will proceed with Du for space computation calculation, 2024-11-27T01:24:18,522 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data3/current/BP-272052841-172.17.0.2-1732670656425/current, will proceed with Du for space computation calculation, 2024-11-27T01:24:18,536 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@47d9f377{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/java.io.tmpdir/jetty-localhost-36991-hadoop-hdfs-3_4_1-tests_jar-_-any-14399164937756368933/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:24:18,537 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b0de5e3{HTTP/1.1, (http/1.1)}{localhost:36991} 2024-11-27T01:24:18,537 INFO [Time-limited test {}] server.Server(415): Started @4816ms 2024-11-27T01:24:18,540 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:24:18,576 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:24:18,577 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:24:18,667 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d65d3803cdb14ac with lease ID 0xa3e95e04fe6f2f93: Processing first storage report for DS-9801fc90-1be0-4920-be50-0b39128a6585 from datanode DatanodeRegistration(127.0.0.1:36269, datanodeUuid=20838706-6b50-4ab4-8a33-a27afc0ef1e7, infoPort=32867, infoSecurePort=0, ipcPort=35757, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425) 2024-11-27T01:24:18,668 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d65d3803cdb14ac with lease ID 0xa3e95e04fe6f2f93: from storage DS-9801fc90-1be0-4920-be50-0b39128a6585 node DatanodeRegistration(127.0.0.1:36269, datanodeUuid=20838706-6b50-4ab4-8a33-a27afc0ef1e7, infoPort=32867, infoSecurePort=0, ipcPort=35757, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T01:24:18,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x451b1d264a5553e0 with lease ID 0xa3e95e04fe6f2f92: Processing first storage report for DS-ab87a46f-5473-416a-b73c-24835b0a27a4 from datanode DatanodeRegistration(127.0.0.1:45195, datanodeUuid=d4665d60-332c-462f-b070-91d408a09e0a, infoPort=42585, infoSecurePort=0, ipcPort=43843, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425) 2024-11-27T01:24:18,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x451b1d264a5553e0 with lease ID 0xa3e95e04fe6f2f92: from storage DS-ab87a46f-5473-416a-b73c-24835b0a27a4 node DatanodeRegistration(127.0.0.1:45195, datanodeUuid=d4665d60-332c-462f-b070-91d408a09e0a, infoPort=42585, infoSecurePort=0, ipcPort=43843, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:24:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d65d3803cdb14ac with lease ID 0xa3e95e04fe6f2f93: Processing first storage report for DS-f44c38a6-1905-4689-a020-485e10d3042a from datanode DatanodeRegistration(127.0.0.1:36269, datanodeUuid=20838706-6b50-4ab4-8a33-a27afc0ef1e7, infoPort=32867, infoSecurePort=0, ipcPort=35757, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425) 2024-11-27T01:24:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d65d3803cdb14ac with lease ID 0xa3e95e04fe6f2f93: from storage DS-f44c38a6-1905-4689-a020-485e10d3042a node DatanodeRegistration(127.0.0.1:36269, datanodeUuid=20838706-6b50-4ab4-8a33-a27afc0ef1e7, infoPort=32867, infoSecurePort=0, ipcPort=35757, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T01:24:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x451b1d264a5553e0 with lease ID 0xa3e95e04fe6f2f92: Processing first storage report for DS-ca565441-cfda-4257-b403-4447c2f6de6d from datanode DatanodeRegistration(127.0.0.1:45195, datanodeUuid=d4665d60-332c-462f-b070-91d408a09e0a, infoPort=42585, infoSecurePort=0, ipcPort=43843, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425) 2024-11-27T01:24:18,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x451b1d264a5553e0 with lease ID 0xa3e95e04fe6f2f92: from storage DS-ca565441-cfda-4257-b403-4447c2f6de6d node DatanodeRegistration(127.0.0.1:45195, datanodeUuid=d4665d60-332c-462f-b070-91d408a09e0a, infoPort=42585, infoSecurePort=0, ipcPort=43843, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:24:18,719 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data5/current/BP-272052841-172.17.0.2-1732670656425/current, will proceed with Du for space computation calculation, 2024-11-27T01:24:18,720 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data6/current/BP-272052841-172.17.0.2-1732670656425/current, will proceed with Du for space computation calculation, 2024-11-27T01:24:18,746 WARN [Thread-123 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:24:18,755 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0580247cf9a23ee with lease ID 0xa3e95e04fe6f2f94: Processing first storage report for DS-42f495a5-2548-46b1-8b0e-84e368707ff8 from datanode DatanodeRegistration(127.0.0.1:37205, datanodeUuid=845fab92-3fee-4427-9122-1c2626f52414, infoPort=44901, infoSecurePort=0, ipcPort=33423, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425) 2024-11-27T01:24:18,756 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0580247cf9a23ee with lease ID 0xa3e95e04fe6f2f94: from storage DS-42f495a5-2548-46b1-8b0e-84e368707ff8 node DatanodeRegistration(127.0.0.1:37205, datanodeUuid=845fab92-3fee-4427-9122-1c2626f52414, infoPort=44901, infoSecurePort=0, ipcPort=33423, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T01:24:18,756 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0580247cf9a23ee with lease ID 0xa3e95e04fe6f2f94: Processing first storage report for DS-2a99acad-a4e6-44fb-bcd6-c0a01d333b41 from datanode DatanodeRegistration(127.0.0.1:37205, datanodeUuid=845fab92-3fee-4427-9122-1c2626f52414, infoPort=44901, infoSecurePort=0, ipcPort=33423, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425) 2024-11-27T01:24:18,757 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0580247cf9a23ee with lease ID 0xa3e95e04fe6f2f94: from storage DS-2a99acad-a4e6-44fb-bcd6-c0a01d333b41 node DatanodeRegistration(127.0.0.1:37205, datanodeUuid=845fab92-3fee-4427-9122-1c2626f52414, infoPort=44901, infoSecurePort=0, ipcPort=33423, storageInfo=lv=-57;cid=testClusterID;nsid=1026910421;c=1732670656425), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:24:18,978 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d 2024-11-27T01:24:19,091 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/zookeeper_0, clientPort=61959, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T01:24:19,103 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61959 2024-11-27T01:24:19,118 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:19,121 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:19,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:24:19,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:24:19,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:24:19,769 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73 with version=8 2024-11-27T01:24:19,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/hbase-staging 2024-11-27T01:24:20,081 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:24:20,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,098 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:24:20,098 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,098 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:24:20,237 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T01:24:20,302 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T01:24:20,312 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T01:24:20,316 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:24:20,345 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5686 (auto-detected) 2024-11-27T01:24:20,346 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T01:24:20,366 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43429 2024-11-27T01:24:20,388 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43429 connecting to ZooKeeper ensemble=127.0.0.1:61959 2024-11-27T01:24:20,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:434290x0, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:24:20,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43429-0x1015706048a0000 connected 2024-11-27T01:24:20,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,457 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,470 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:24:20,475 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73, hbase.cluster.distributed=false 2024-11-27T01:24:20,498 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:24:20,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43429 2024-11-27T01:24:20,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43429 2024-11-27T01:24:20,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43429 2024-11-27T01:24:20,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43429 2024-11-27T01:24:20,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43429 2024-11-27T01:24:20,625 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:24:20,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,627 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:24:20,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,628 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:24:20,630 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:24:20,633 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:24:20,634 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34199 2024-11-27T01:24:20,635 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34199 connecting to ZooKeeper ensemble=127.0.0.1:61959 2024-11-27T01:24:20,637 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,639 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341990x0, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:24:20,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:341990x0, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:24:20,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34199-0x1015706048a0001 connected 2024-11-27T01:24:20,651 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:24:20,659 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:24:20,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:24:20,667 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:24:20,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34199 2024-11-27T01:24:20,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34199 2024-11-27T01:24:20,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34199 2024-11-27T01:24:20,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34199 2024-11-27T01:24:20,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34199 2024-11-27T01:24:20,687 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:24:20,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,688 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:24:20,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,688 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:24:20,689 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:24:20,689 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:24:20,690 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42983 2024-11-27T01:24:20,691 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42983 connecting to ZooKeeper ensemble=127.0.0.1:61959 2024-11-27T01:24:20,692 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,694 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429830x0, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:24:20,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42983-0x1015706048a0002 connected 2024-11-27T01:24:20,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:24:20,701 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:24:20,702 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:24:20,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:24:20,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:24:20,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42983 2024-11-27T01:24:20,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42983 2024-11-27T01:24:20,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42983 2024-11-27T01:24:20,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42983 2024-11-27T01:24:20,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42983 2024-11-27T01:24:20,728 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:24:20,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,729 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:24:20,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:24:20,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:24:20,730 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:24:20,730 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:24:20,731 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39619 2024-11-27T01:24:20,733 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39619 connecting to ZooKeeper ensemble=127.0.0.1:61959 2024-11-27T01:24:20,734 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,737 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396190x0, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:24:20,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39619-0x1015706048a0003 connected 2024-11-27T01:24:20,744 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:24:20,745 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:24:20,745 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:24:20,746 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:24:20,748 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:24:20,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39619 2024-11-27T01:24:20,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39619 2024-11-27T01:24:20,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39619 2024-11-27T01:24:20,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39619 2024-11-27T01:24:20,751 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39619 2024-11-27T01:24:20,766 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7d6f26a4c136:43429 2024-11-27T01:24:20,766 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:20,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,776 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:24:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:24:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:24:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,799 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T01:24:20,800 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7d6f26a4c136,43429,1732670659874 from backup master directory 2024-11-27T01:24:20,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:20,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:24:20,804 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:24:20,804 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:20,806 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T01:24:20,807 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T01:24:20,868 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/hbase.id] with ID: be445ce8-d3ee-4cc8-bc38-06bc3b180362 2024-11-27T01:24:20,868 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/.tmp/hbase.id 2024-11-27T01:24:20,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:24:20,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:24:20,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:24:20,883 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/.tmp/hbase.id]:[hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/hbase.id] 2024-11-27T01:24:20,926 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:24:20,931 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T01:24:20,951 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-27T01:24:20,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:20,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:24:20,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:24:20,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:24:20,994 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:24:20,997 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T01:24:21,003 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:21,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:24:21,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:24:21,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:24:21,058 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store 2024-11-27T01:24:21,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:24:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:24:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:24:21,086 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T01:24:21,089 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:21,090 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:24:21,090 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:21,090 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:21,091 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:24:21,092 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:21,092 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:21,093 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732670661090Disabling compacts and flushes for region at 1732670661090Disabling writes for close at 1732670661091 (+1 ms)Writing region close event to WAL at 1732670661092 (+1 ms)Closed at 1732670661092 2024-11-27T01:24:21,094 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/.initializing 2024-11-27T01:24:21,095 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/WALs/7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:21,102 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:21,116 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C43429%2C1732670659874, suffix=, logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/WALs/7d6f26a4c136,43429,1732670659874, archiveDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/oldWALs, maxLogs=10 2024-11-27T01:24:21,149 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/WALs/7d6f26a4c136,43429,1732670659874/7d6f26a4c136%2C43429%2C1732670659874.1732670661121, exclude list is [], retry=0 2024-11-27T01:24:21,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:21,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:21,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:21,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:21,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T01:24:21,218 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/WALs/7d6f26a4c136,43429,1732670659874/7d6f26a4c136%2C43429%2C1732670659874.1732670661121 2024-11-27T01:24:21,219 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867)] 2024-11-27T01:24:21,219 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:21,220 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:21,223 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,225 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T01:24:21,292 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:21,295 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,298 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T01:24:21,298 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:21,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T01:24:21,302 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,302 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:21,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T01:24:21,305 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,306 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:21,306 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,310 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,311 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,315 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,316 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,319 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:24:21,323 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:24:21,328 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:21,329 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62505874, jitterRate=-0.06858989596366882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:24:21,334 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732670661237Initializing all the Stores at 1732670661239 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670661240 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670661240Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670661241 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670661241Cleaning up temporary data from old regions at 1732670661316 (+75 ms)Region opened successfully at 1732670661334 (+18 ms) 2024-11-27T01:24:21,335 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T01:24:21,372 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696e742e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:24:21,408 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T01:24:21,420 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T01:24:21,420 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T01:24:21,424 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:24:21,425 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T01:24:21,430 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T01:24:21,430 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T01:24:21,458 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T01:24:21,466 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T01:24:21,469 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T01:24:21,472 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T01:24:21,473 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T01:24:21,477 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T01:24:21,479 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T01:24:21,483 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T01:24:21,484 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T01:24:21,485 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T01:24:21,487 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T01:24:21,505 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T01:24:21,506 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,515 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7d6f26a4c136,43429,1732670659874, sessionid=0x1015706048a0000, setting cluster-up flag (Was=false) 2024-11-27T01:24:21,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,537 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T01:24:21,538 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:21,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:21,552 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T01:24:21,553 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:21,560 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T01:24:21,653 INFO [AsyncFSWAL-0-hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData-prefix:7d6f26a4c136,43429,1732670659874 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-27T01:24:21,655 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(746): ClusterId : be445ce8-d3ee-4cc8-bc38-06bc3b180362 2024-11-27T01:24:21,655 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(746): ClusterId : be445ce8-d3ee-4cc8-bc38-06bc3b180362 2024-11-27T01:24:21,656 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(746): ClusterId : be445ce8-d3ee-4cc8-bc38-06bc3b180362 2024-11-27T01:24:21,659 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:24:21,659 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:24:21,659 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:24:21,667 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:24:21,667 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:24:21,667 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:24:21,667 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:24:21,667 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:24:21,667 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:24:21,671 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:24:21,671 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:24:21,671 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:24:21,671 DEBUG [RS:1;7d6f26a4c136:42983 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56354f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:24:21,672 DEBUG [RS:2;7d6f26a4c136:39619 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa7c2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:24:21,672 DEBUG [RS:0;7d6f26a4c136:34199 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea9f955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:24:21,674 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:24:21,687 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T01:24:21,693 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7d6f26a4c136:34199 2024-11-27T01:24:21,695 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7d6f26a4c136:39619 2024-11-27T01:24:21,696 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T01:24:21,697 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:24:21,697 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:24:21,697 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:24:21,697 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:24:21,697 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:24:21,697 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:24:21,698 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7d6f26a4c136:42983 2024-11-27T01:24:21,698 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:24:21,698 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:24:21,698 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:24:21,701 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,43429,1732670659874 with port=39619, startcode=1732670660728 2024-11-27T01:24:21,701 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,43429,1732670659874 with port=42983, startcode=1732670660687 2024-11-27T01:24:21,701 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,43429,1732670659874 with port=34199, startcode=1732670660585 2024-11-27T01:24:21,704 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7d6f26a4c136,43429,1732670659874 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T01:24:21,713 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:24:21,713 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:24:21,713 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:24:21,713 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:24:21,714 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7d6f26a4c136:0, corePoolSize=10, maxPoolSize=10 2024-11-27T01:24:21,714 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,714 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:24:21,714 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,714 DEBUG [RS:0;7d6f26a4c136:34199 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:24:21,714 DEBUG [RS:1;7d6f26a4c136:42983 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:24:21,714 DEBUG [RS:2;7d6f26a4c136:39619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:24:21,723 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:24:21,723 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T01:24:21,729 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670691729 2024-11-27T01:24:21,730 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T01:24:21,731 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,731 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T01:24:21,731 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T01:24:21,736 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T01:24:21,736 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T01:24:21,737 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T01:24:21,737 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T01:24:21,738 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,742 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T01:24:21,743 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T01:24:21,744 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T01:24:21,749 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T01:24:21,765 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T01:24:21,768 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670661767,5,FailOnTimeoutGroup] 2024-11-27T01:24:21,770 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670661768,5,FailOnTimeoutGroup] 2024-11-27T01:24:21,770 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,772 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T01:24:21,775 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55447, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:24:21,775 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33143, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:24:21,776 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60735, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:24:21,776 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,777 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:24:21,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:24:21,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:24:21,780 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T01:24:21,781 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73 2024-11-27T01:24:21,782 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:21,785 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43429 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:21,799 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:21,800 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43429 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:21,804 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73 2024-11-27T01:24:21,805 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41069 2024-11-27T01:24:21,805 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:24:21,806 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:21,806 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43429 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:21,808 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73 2024-11-27T01:24:21,808 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41069 2024-11-27T01:24:21,808 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:24:21,810 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73 2024-11-27T01:24:21,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:24:21,811 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41069 2024-11-27T01:24:21,811 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:24:21,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:24:21,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:24:21,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:24:21,815 DEBUG [RS:1;7d6f26a4c136:42983 {}] zookeeper.ZKUtil(111): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:21,816 WARN [RS:1;7d6f26a4c136:42983 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:24:21,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:21,816 INFO [RS:1;7d6f26a4c136:42983 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:21,816 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:21,817 DEBUG [RS:0;7d6f26a4c136:34199 {}] zookeeper.ZKUtil(111): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:21,817 WARN [RS:0;7d6f26a4c136:34199 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:24:21,818 DEBUG [RS:2;7d6f26a4c136:39619 {}] zookeeper.ZKUtil(111): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:21,818 INFO [RS:0;7d6f26a4c136:34199 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:21,818 WARN [RS:2;7d6f26a4c136:39619 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:24:21,818 INFO [RS:2;7d6f26a4c136:39619 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:21,818 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:21,818 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:21,820 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,42983,1732670660687] 2024-11-27T01:24:21,820 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,34199,1732670660585] 2024-11-27T01:24:21,820 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,39619,1732670660728] 2024-11-27T01:24:21,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:24:21,824 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:24:21,824 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:21,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:24:21,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:24:21,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:21,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:24:21,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:24:21,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:21,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:24:21,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:24:21,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:21,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:21,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:24:21,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740 2024-11-27T01:24:21,849 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740 2024-11-27T01:24:21,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:24:21,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:24:21,854 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:24:21,854 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:24:21,854 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:24:21,855 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:24:21,858 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:24:21,865 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:21,867 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65341188, jitterRate=-0.02634042501449585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:24:21,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732670661816Initializing all the Stores at 1732670661819 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670661819Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670661821 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670661821Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670661821Cleaning up temporary data from old regions at 1732670661854 (+33 ms)Region opened successfully at 1732670661870 (+16 ms) 2024-11-27T01:24:21,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:24:21,870 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:24:21,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:24:21,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:24:21,871 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:24:21,872 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:24:21,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732670661870Disabling compacts and flushes for region at 1732670661870Disabling writes for close at 1732670661870Writing region close event to WAL at 1732670661871 (+1 ms)Closed at 1732670661872 (+1 ms) 2024-11-27T01:24:21,876 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:24:21,876 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T01:24:21,878 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:24:21,879 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:24:21,879 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:24:21,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T01:24:21,885 INFO [RS:2;7d6f26a4c136:39619 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:24:21,885 INFO [RS:0;7d6f26a4c136:34199 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:24:21,885 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,885 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,886 INFO [RS:1;7d6f26a4c136:42983 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:24:21,886 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,887 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:24:21,887 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:24:21,887 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:24:21,895 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:24:21,895 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:24:21,897 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,897 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,897 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,897 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:24:21,897 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,897 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,898 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,898 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,898 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:24:21,898 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,898 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,898 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,899 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,899 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,899 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,899 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:24:21,899 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,899 DEBUG [RS:0;7d6f26a4c136:34199 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:24:21,899 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,900 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,900 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:24:21,900 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,900 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,900 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,900 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,900 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,901 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,901 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,901 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:24:21,902 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T01:24:21,902 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,902 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,902 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,902 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,902 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,902 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,903 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:24:21,903 DEBUG [RS:1;7d6f26a4c136:42983 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:24:21,908 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,908 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:24:21,909 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,909 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,910 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,910 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,910 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,910 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:24:21,910 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:24:21,910 DEBUG [RS:2;7d6f26a4c136:39619 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:24:21,916 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,917 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,917 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,917 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,917 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,918 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,34199,1732670660585-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:24:21,920 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,920 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,920 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,921 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,921 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,921 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,42983,1732670660687-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:24:21,927 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,927 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,927 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,928 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,928 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,928 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,39619,1732670660728-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:24:21,948 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:24:21,950 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,34199,1732670660585-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,951 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,951 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.Replication(171): 7d6f26a4c136,34199,1732670660585 started 2024-11-27T01:24:21,952 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:24:21,953 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,42983,1732670660687-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,953 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,953 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.Replication(171): 7d6f26a4c136,42983,1732670660687 started 2024-11-27T01:24:21,953 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:24:21,954 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,39619,1732670660728-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,954 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,954 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.Replication(171): 7d6f26a4c136,39619,1732670660728 started 2024-11-27T01:24:21,979 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,979 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,979 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,42983,1732670660687, RpcServer on 7d6f26a4c136/172.17.0.2:42983, sessionid=0x1015706048a0002 2024-11-27T01:24:21,979 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:21,980 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,34199,1732670660585, RpcServer on 7d6f26a4c136/172.17.0.2:34199, sessionid=0x1015706048a0001 2024-11-27T01:24:21,979 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,39619,1732670660728, RpcServer on 7d6f26a4c136/172.17.0.2:39619, sessionid=0x1015706048a0003 2024-11-27T01:24:21,980 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:24:21,980 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:24:21,980 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:24:21,980 DEBUG [RS:0;7d6f26a4c136:34199 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:21,980 DEBUG [RS:2;7d6f26a4c136:39619 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:21,981 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,34199,1732670660585' 2024-11-27T01:24:21,981 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,39619,1732670660728' 2024-11-27T01:24:21,981 DEBUG [RS:1;7d6f26a4c136:42983 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:21,981 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:24:21,981 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:24:21,981 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,42983,1732670660687' 2024-11-27T01:24:21,981 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:24:21,982 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:24:21,982 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:24:21,982 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:24:21,983 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:24:21,983 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:24:21,983 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:24:21,983 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:24:21,983 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:24:21,983 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:24:21,983 DEBUG [RS:1;7d6f26a4c136:42983 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:21,983 DEBUG [RS:0;7d6f26a4c136:34199 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:21,983 DEBUG [RS:2;7d6f26a4c136:39619 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:21,983 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,42983,1732670660687' 2024-11-27T01:24:21,983 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,34199,1732670660585' 2024-11-27T01:24:21,983 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,39619,1732670660728' 2024-11-27T01:24:21,983 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:24:21,983 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:24:21,983 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:24:21,984 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:24:21,984 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:24:21,985 DEBUG [RS:1;7d6f26a4c136:42983 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:24:21,985 DEBUG [RS:0;7d6f26a4c136:34199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:24:21,986 INFO [RS:1;7d6f26a4c136:42983 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:24:21,986 INFO [RS:0;7d6f26a4c136:34199 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:24:21,986 INFO [RS:1;7d6f26a4c136:42983 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:24:21,986 INFO [RS:0;7d6f26a4c136:34199 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:24:21,987 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:24:21,988 DEBUG [RS:2;7d6f26a4c136:39619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:24:21,988 INFO [RS:2;7d6f26a4c136:39619 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:24:21,989 INFO [RS:2;7d6f26a4c136:39619 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:24:22,053 WARN [7d6f26a4c136:43429 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T01:24:22,092 INFO [RS:2;7d6f26a4c136:39619 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:22,093 INFO [RS:1;7d6f26a4c136:42983 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:22,093 INFO [RS:0;7d6f26a4c136:34199 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:22,098 INFO [RS:0;7d6f26a4c136:34199 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C34199%2C1732670660585, suffix=, logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,34199,1732670660585, archiveDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs, maxLogs=32 2024-11-27T01:24:22,100 INFO [RS:2;7d6f26a4c136:39619 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C39619%2C1732670660728, suffix=, logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,39619,1732670660728, archiveDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs, maxLogs=32 2024-11-27T01:24:22,105 INFO [RS:1;7d6f26a4c136:42983 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C42983%2C1732670660687, suffix=, logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687, archiveDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs, maxLogs=32 2024-11-27T01:24:22,126 DEBUG [RS:0;7d6f26a4c136:34199 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,34199,1732670660585/7d6f26a4c136%2C34199%2C1732670660585.1732670662106, exclude list is [], retry=0 2024-11-27T01:24:22,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:22,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:22,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:22,135 DEBUG [RS:2;7d6f26a4c136:39619 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,39619,1732670660728/7d6f26a4c136%2C39619%2C1732670660728.1732670662103, exclude list is [], retry=0 2024-11-27T01:24:22,135 DEBUG [RS:1;7d6f26a4c136:42983 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687/7d6f26a4c136%2C42983%2C1732670660687.1732670662108, exclude list is [], retry=0 2024-11-27T01:24:22,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:22,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:22,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:22,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:22,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:22,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:22,149 INFO [RS:0;7d6f26a4c136:34199 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,34199,1732670660585/7d6f26a4c136%2C34199%2C1732670660585.1732670662106 2024-11-27T01:24:22,154 DEBUG [RS:0;7d6f26a4c136:34199 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:22,161 INFO [RS:2;7d6f26a4c136:39619 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,39619,1732670660728/7d6f26a4c136%2C39619%2C1732670660728.1732670662103 2024-11-27T01:24:22,162 DEBUG [RS:2;7d6f26a4c136:39619 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:22,163 INFO [RS:1;7d6f26a4c136:42983 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 2024-11-27T01:24:22,168 DEBUG [RS:1;7d6f26a4c136:42983 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:22,305 DEBUG [7d6f26a4c136:43429 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-27T01:24:22,313 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:24:22,320 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:24:22,320 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:24:22,320 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T01:24:22,320 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:24:22,320 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:24:22,320 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T01:24:22,320 INFO [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:24:22,320 INFO [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:24:22,320 INFO [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T01:24:22,320 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:24:22,328 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:22,334 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,39619,1732670660728, state=OPENING 2024-11-27T01:24:22,339 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T01:24:22,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:22,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:22,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:22,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:22,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,343 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:24:22,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,39619,1732670660728}] 2024-11-27T01:24:22,521 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:24:22,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48679, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:24:22,537 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T01:24:22,538 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:22,538 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T01:24:22,542 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C39619%2C1732670660728.meta, suffix=.meta, logDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,39619,1732670660728, archiveDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs, maxLogs=32 2024-11-27T01:24:22,558 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,39619,1732670660728/7d6f26a4c136%2C39619%2C1732670660728.meta.1732670662543.meta, exclude list is [], retry=0 2024-11-27T01:24:22,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:22,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:22,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:22,565 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,39619,1732670660728/7d6f26a4c136%2C39619%2C1732670660728.meta.1732670662543.meta 2024-11-27T01:24:22,566 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:22,566 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:22,567 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T01:24:22,570 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T01:24:22,575 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T01:24:22,579 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T01:24:22,580 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:22,580 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T01:24:22,580 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T01:24:22,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:24:22,584 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:24:22,584 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:22,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:22,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:24:22,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:24:22,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:22,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:22,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:24:22,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:24:22,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:22,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:22,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:24:22,591 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:24:22,591 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:22,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:24:22,592 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:24:22,593 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740 2024-11-27T01:24:22,596 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740 2024-11-27T01:24:22,598 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:24:22,598 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:24:22,599 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:24:22,601 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:24:22,603 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69435798, jitterRate=0.0346740186214447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:24:22,603 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T01:24:22,604 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732670662580Writing region info on filesystem at 1732670662581 (+1 ms)Initializing all the Stores at 1732670662582 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670662582Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670662582Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670662583 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670662583Cleaning up temporary data from old regions at 1732670662598 (+15 ms)Running coprocessor post-open hooks at 1732670662603 (+5 ms)Region opened successfully at 1732670662604 (+1 ms) 2024-11-27T01:24:22,612 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732670662511 2024-11-27T01:24:22,623 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T01:24:22,624 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T01:24:22,626 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:22,628 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,39619,1732670660728, state=OPEN 2024-11-27T01:24:22,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:24:22,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:24:22,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:24:22,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:24:22,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,632 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:24:22,632 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:22,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T01:24:22,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,39619,1732670660728 in 287 msec 2024-11-27T01:24:22,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T01:24:22,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 755 msec 2024-11-27T01:24:22,645 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:24:22,646 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T01:24:22,694 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:24:22,696 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,39619,1732670660728, seqNum=-1] 2024-11-27T01:24:22,726 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:24:22,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42957, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:24:22,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1570 sec 2024-11-27T01:24:22,747 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732670662747, completionTime=-1 2024-11-27T01:24:22,749 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-27T01:24:22,750 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:24:22,775 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-27T01:24:22,775 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670722775 2024-11-27T01:24:22,775 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670782775 2024-11-27T01:24:22,775 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-27T01:24:22,777 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:24:22,782 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43429,1732670659874-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:22,783 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43429,1732670659874-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:22,783 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43429,1732670659874-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:22,784 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7d6f26a4c136:43429, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:22,785 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:22,785 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:22,791 DEBUG [master/7d6f26a4c136:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T01:24:22,815 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.011sec 2024-11-27T01:24:22,817 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T01:24:22,818 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T01:24:22,819 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T01:24:22,819 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T01:24:22,819 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T01:24:22,820 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43429,1732670659874-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:24:22,821 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43429,1732670659874-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T01:24:22,825 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T01:24:22,826 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T01:24:22,826 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43429,1732670659874-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:22,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3c67d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:24:22,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,43429,-1 for getting cluster id 2024-11-27T01:24:22,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:24:22,896 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'be445ce8-d3ee-4cc8-bc38-06bc3b180362' 2024-11-27T01:24:22,899 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:24:22,899 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "be445ce8-d3ee-4cc8-bc38-06bc3b180362" 2024-11-27T01:24:22,899 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e6fdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:24:22,899 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,43429,-1] 2024-11-27T01:24:22,902 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:24:22,904 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:22,905 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:24:22,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ab4471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:24:22,908 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:24:22,916 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,39619,1732670660728, seqNum=-1] 2024-11-27T01:24:22,916 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:24:22,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:24:22,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:22,940 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:41069/hbase 2024-11-27T01:24:22,963 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=361, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=6791 2024-11-27T01:24:22,990 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:22,994 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:22,995 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:23,001 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57948499, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-57948499, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:23,017 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57948499/hregion-57948499.1732670663002, exclude list is [], retry=0 2024-11-27T01:24:23,021 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:23,022 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:23,022 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:23,025 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57948499/hregion-57948499.1732670663002 2024-11-27T01:24:23,025 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:23,026 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28dae1a93b45e5db7ad9354595564047, NAME => 'testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741838_1014 (size=64) 2024-11-27T01:24:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741838_1014 (size=64) 2024-11-27T01:24:23,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741838_1014 (size=64) 2024-11-27T01:24:23,044 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:23,046 INFO [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,049 INFO [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28dae1a93b45e5db7ad9354595564047 columnFamilyName a 2024-11-27T01:24:23,049 DEBUG [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:23,050 INFO [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] regionserver.HStore(327): Store=28dae1a93b45e5db7ad9354595564047/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:23,050 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,051 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,052 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,052 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,053 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,055 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,059 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:23,060 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28dae1a93b45e5db7ad9354595564047; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60539301, jitterRate=-0.09789411723613739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:23,061 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28dae1a93b45e5db7ad9354595564047: Writing region info on filesystem at 1732670663044Initializing all the Stores at 1732670663046 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670663046Cleaning up temporary data from old regions at 1732670663053 (+7 ms)Region opened successfully at 1732670663061 (+8 ms) 2024-11-27T01:24:23,061 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28dae1a93b45e5db7ad9354595564047, disabling compactions & flushes 2024-11-27T01:24:23,061 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047. 2024-11-27T01:24:23,061 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047. 2024-11-27T01:24:23,061 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047. after waiting 0 ms 2024-11-27T01:24:23,061 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047. 2024-11-27T01:24:23,062 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047. 2024-11-27T01:24:23,062 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28dae1a93b45e5db7ad9354595564047: Waiting for close lock at 1732670663061Disabling compacts and flushes for region at 1732670663061Disabling writes for close at 1732670663061Writing region close event to WAL at 1732670663062 (+1 ms)Closed at 1732670663062 2024-11-27T01:24:23,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741837_1013 (size=95) 2024-11-27T01:24:23,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741837_1013 (size=95) 2024-11-27T01:24:23,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741837_1013 (size=95) 2024-11-27T01:24:23,074 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:23,074 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57948499:(num 1732670663002) 2024-11-27T01:24:23,076 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:24:23,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741839_1015 (size=320) 2024-11-27T01:24:23,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741839_1015 (size=320) 2024-11-27T01:24:23,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741839_1015 (size=320) 2024-11-27T01:24:23,093 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:24:23,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741840_1016 (size=253) 2024-11-27T01:24:23,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741840_1016 (size=253) 2024-11-27T01:24:23,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741840_1016 (size=253) 2024-11-27T01:24:23,128 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1, size=320 (320bytes) 2024-11-27T01:24:23,129 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-27T01:24:23,129 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-27T01:24:23,129 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1 2024-11-27T01:24:23,134 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1 after 4ms 2024-11-27T01:24:23,140 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,141 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1 took 14ms 2024-11-27T01:24:23,151 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1 so closing down 2024-11-27T01:24:23,151 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:23,154 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T01:24:23,156 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T01:24:23,158 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:23,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741841_1017 (size=320) 2024-11-27T01:24:23,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741841_1017 (size=320) 2024-11-27T01:24:23,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741841_1017 (size=320) 2024-11-27T01:24:23,175 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:23,178 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002 2024-11-27T01:24:23,182 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-27T01:24:23,182 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1, journal: Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1, size=320 (320bytes) at 1732670663128Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1 so closing down at 1732670663151 (+23 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000001-wal-1.temp at 1732670663156 (+5 ms)3 split writer threads finished at 1732670663158 (+2 ms)Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732670663176 (+18 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002 at 1732670663178 (+2 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732670663182 (+4 ms) 2024-11-27T01:24:23,199 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2, size=253 (253bytes) 2024-11-27T01:24:23,200 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2 2024-11-27T01:24:23,200 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2 after 0ms 2024-11-27T01:24:23,205 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,205 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2 took 6ms 2024-11-27T01:24:23,208 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2 so closing down 2024-11-27T01:24:23,208 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:23,212 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T01:24:23,214 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T01:24:23,215 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:23,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741842_1018 (size=253) 2024-11-27T01:24:23,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741842_1018 (size=253) 2024-11-27T01:24:23,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741842_1018 (size=253) 2024-11-27T01:24:23,227 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:23,232 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,234 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-27T01:24:23,236 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-27T01:24:23,236 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2, journal: Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2, size=253 (253bytes) at 1732670663199Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2 so closing down at 1732670663208 (+9 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002-wal-2.temp at 1732670663214 (+6 ms)3 split writer threads finished at 1732670663215 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732670663227 (+12 ms)Processed 1 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732670663236 (+9 ms) 2024-11-27T01:24:23,237 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:23,239 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:23,254 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal.1732670663240, exclude list is [], retry=0 2024-11-27T01:24:23,258 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:23,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:23,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:23,261 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal.1732670663240 2024-11-27T01:24:23,262 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867)] 2024-11-27T01:24:23,262 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28dae1a93b45e5db7ad9354595564047, NAME => 'testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:23,262 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:23,262 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,262 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,264 INFO [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,266 INFO [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28dae1a93b45e5db7ad9354595564047 columnFamilyName a 2024-11-27T01:24:23,266 DEBUG [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:23,267 INFO [StoreOpener-28dae1a93b45e5db7ad9354595564047-1 {}] regionserver.HStore(327): Store=28dae1a93b45e5db7ad9354595564047/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:23,267 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,273 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002 2024-11-27T01:24:23,277 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,283 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002 2024-11-27T01:24:23,286 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28dae1a93b45e5db7ad9354595564047 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T01:24:23,341 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/.tmp/a/2917b162cda74356889ae10e42e6ac6a is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732670663074/Put/seqid=0 2024-11-27T01:24:23,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741844_1020 (size=5170) 2024-11-27T01:24:23,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741844_1020 (size=5170) 2024-11-27T01:24:23,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741844_1020 (size=5170) 2024-11-27T01:24:23,356 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/.tmp/a/2917b162cda74356889ae10e42e6ac6a 2024-11-27T01:24:23,403 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/.tmp/a/2917b162cda74356889ae10e42e6ac6a as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/a/2917b162cda74356889ae10e42e6ac6a 2024-11-27T01:24:23,413 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/a/2917b162cda74356889ae10e42e6ac6a, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T01:24:23,420 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 28dae1a93b45e5db7ad9354595564047 in 132ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T01:24:23,422 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/0000000000000000002 2024-11-27T01:24:23,423 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,423 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,427 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28dae1a93b45e5db7ad9354595564047 2024-11-27T01:24:23,431 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/28dae1a93b45e5db7ad9354595564047/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T01:24:23,432 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28dae1a93b45e5db7ad9354595564047; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67902944, jitterRate=0.011832714080810547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:23,433 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28dae1a93b45e5db7ad9354595564047: Writing region info on filesystem at 1732670663262Initializing all the Stores at 1732670663264 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670663264Obtaining lock to block concurrent updates at 1732670663286 (+22 ms)Preparing flush snapshotting stores in 28dae1a93b45e5db7ad9354595564047 at 1732670663286Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732670663289 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732670662991.28dae1a93b45e5db7ad9354595564047. at 1732670663289Flushing 28dae1a93b45e5db7ad9354595564047/a: creating writer at 1732670663290 (+1 ms)Flushing 28dae1a93b45e5db7ad9354595564047/a: appending metadata at 1732670663330 (+40 ms)Flushing 28dae1a93b45e5db7ad9354595564047/a: closing flushed file at 1732670663333 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61cb8a35: reopening flushed file at 1732670663401 (+68 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 28dae1a93b45e5db7ad9354595564047 in 132ms, sequenceid=2, compaction requested=false; wal=null at 1732670663420 (+19 ms)Cleaning up temporary data from old regions at 1732670663423 (+3 ms)Region opened successfully at 1732670663433 (+10 ms) 2024-11-27T01:24:23,464 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=372 (was 361) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:51320 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41069/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57322 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:51146 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57028 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:56940 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57160 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=6762 (was 6791) 2024-11-27T01:24:23,476 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=372, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=6760 2024-11-27T01:24:23,498 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:23,501 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:23,502 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:23,507 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44328380, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-44328380, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:23,526 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44328380/hregion-44328380.1732670663508, exclude list is [], retry=0 2024-11-27T01:24:23,531 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:23,532 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:23,533 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:23,540 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44328380/hregion-44328380.1732670663508 2024-11-27T01:24:23,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:23,541 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0556937df467ac954a4e560539f4bf53, NAME => 'testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:23,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741846_1022 (size=64) 2024-11-27T01:24:23,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741846_1022 (size=64) 2024-11-27T01:24:23,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741846_1022 (size=64) 2024-11-27T01:24:23,558 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:23,563 INFO [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,566 INFO [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0556937df467ac954a4e560539f4bf53 columnFamilyName a 2024-11-27T01:24:23,566 DEBUG [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:23,567 INFO [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] regionserver.HStore(327): Store=0556937df467ac954a4e560539f4bf53/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:23,567 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,570 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,570 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,572 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,575 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:23,576 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0556937df467ac954a4e560539f4bf53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74034542, jitterRate=0.10320064425468445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:23,576 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0556937df467ac954a4e560539f4bf53: Writing region info on filesystem at 1732670663559Initializing all the Stores at 1732670663560 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670663561 (+1 ms)Cleaning up temporary data from old regions at 1732670663570 (+9 ms)Region opened successfully at 1732670663576 (+6 ms) 2024-11-27T01:24:23,576 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0556937df467ac954a4e560539f4bf53, disabling compactions & flushes 2024-11-27T01:24:23,576 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53. 2024-11-27T01:24:23,577 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53. 2024-11-27T01:24:23,577 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53. after waiting 0 ms 2024-11-27T01:24:23,577 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53. 2024-11-27T01:24:23,577 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53. 2024-11-27T01:24:23,577 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0556937df467ac954a4e560539f4bf53: Waiting for close lock at 1732670663576Disabling compacts and flushes for region at 1732670663576Disabling writes for close at 1732670663577 (+1 ms)Writing region close event to WAL at 1732670663577Closed at 1732670663577 2024-11-27T01:24:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741845_1021 (size=95) 2024-11-27T01:24:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741845_1021 (size=95) 2024-11-27T01:24:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741845_1021 (size=95) 2024-11-27T01:24:23,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:23,587 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44328380:(num 1732670663508) 2024-11-27T01:24:23,589 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:24:23,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741847_1023 (size=320) 2024-11-27T01:24:23,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741847_1023 (size=320) 2024-11-27T01:24:23,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741847_1023 (size=320) 2024-11-27T01:24:23,605 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:24:23,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741848_1024 (size=253) 2024-11-27T01:24:23,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741848_1024 (size=253) 2024-11-27T01:24:23,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741848_1024 (size=253) 2024-11-27T01:24:23,645 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2, size=253 (253bytes) 2024-11-27T01:24:23,645 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2 2024-11-27T01:24:23,647 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2 after 1ms 2024-11-27T01:24:23,651 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,651 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2 took 6ms 2024-11-27T01:24:23,655 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2 so closing down 2024-11-27T01:24:23,655 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:23,657 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T01:24:23,659 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T01:24:23,659 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:23,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741849_1025 (size=253) 2024-11-27T01:24:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741849_1025 (size=253) 2024-11-27T01:24:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741849_1025 (size=253) 2024-11-27T01:24:23,678 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:23,681 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002 2024-11-27T01:24:23,681 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-27T01:24:23,681 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2, journal: Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2, size=253 (253bytes) at 1732670663645Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2 so closing down at 1732670663655 (+10 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002-wal-2.temp at 1732670663659 (+4 ms)3 split writer threads finished at 1732670663659Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732670663678 (+19 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002 at 1732670663681 (+3 ms)Processed 1 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1732670663681 2024-11-27T01:24:23,697 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1, size=320 (320bytes) 2024-11-27T01:24:23,697 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1 2024-11-27T01:24:23,697 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1 after 0ms 2024-11-27T01:24:23,701 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,702 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1 took 6ms 2024-11-27T01:24:23,705 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1 so closing down 2024-11-27T01:24:23,705 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:23,708 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T01:24:23,710 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T01:24:23,710 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:23,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741850_1026 (size=320) 2024-11-27T01:24:23,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741850_1026 (size=320) 2024-11-27T01:24:23,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741850_1026 (size=320) 2024-11-27T01:24:23,736 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:23,741 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,744 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002, length=253 2024-11-27T01:24:23,747 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002 2024-11-27T01:24:23,747 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 45 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-27T01:24:23,748 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1, journal: Splitting hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1, size=320 (320bytes) at 1732670663697Finishing writing output for hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1 so closing down at 1732670663705 (+8 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000001-wal-1.temp at 1732670663710 (+5 ms)3 split writer threads finished at 1732670663710Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732670663736 (+26 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002 at 1732670663747 (+11 ms)Processed 2 edits across 1 Regions in 45 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1732670663747 2024-11-27T01:24:23,748 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:23,751 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:23,771 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal.1732670663752, exclude list is [], retry=0 2024-11-27T01:24:23,775 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:23,775 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:23,776 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:23,781 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal.1732670663752 2024-11-27T01:24:23,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:23,782 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0556937df467ac954a4e560539f4bf53, NAME => 'testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:23,782 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:23,782 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,782 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,784 INFO [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,786 INFO [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0556937df467ac954a4e560539f4bf53 columnFamilyName a 2024-11-27T01:24:23,786 DEBUG [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:23,797 INFO [StoreOpener-0556937df467ac954a4e560539f4bf53-1 {}] regionserver.HStore(327): Store=0556937df467ac954a4e560539f4bf53/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:23,800 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,802 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,805 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,810 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002 2024-11-27T01:24:23,815 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:23,818 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002 2024-11-27T01:24:23,818 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0556937df467ac954a4e560539f4bf53 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T01:24:23,843 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/.tmp/a/ed4b19f1d97a4d7683fbb541f58379ce is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732670663587/Put/seqid=0 2024-11-27T01:24:23,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741852_1028 (size=5170) 2024-11-27T01:24:23,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741852_1028 (size=5170) 2024-11-27T01:24:23,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741852_1028 (size=5170) 2024-11-27T01:24:23,855 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/.tmp/a/ed4b19f1d97a4d7683fbb541f58379ce 2024-11-27T01:24:23,864 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/.tmp/a/ed4b19f1d97a4d7683fbb541f58379ce as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/a/ed4b19f1d97a4d7683fbb541f58379ce 2024-11-27T01:24:23,872 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/a/ed4b19f1d97a4d7683fbb541f58379ce, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T01:24:23,872 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0556937df467ac954a4e560539f4bf53 in 54ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T01:24:23,873 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/0000000000000000002 2024-11-27T01:24:23,874 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,874 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,876 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0556937df467ac954a4e560539f4bf53 2024-11-27T01:24:23,879 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/0556937df467ac954a4e560539f4bf53/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T01:24:23,880 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0556937df467ac954a4e560539f4bf53; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64560809, jitterRate=-0.03796897828578949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:23,881 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0556937df467ac954a4e560539f4bf53: Writing region info on filesystem at 1732670663782Initializing all the Stores at 1732670663783 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670663783Obtaining lock to block concurrent updates at 1732670663818 (+35 ms)Preparing flush snapshotting stores in 0556937df467ac954a4e560539f4bf53 at 1732670663818Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732670663818Flushing stores of testReplayEditsWrittenIntoWAL,,1732670663499.0556937df467ac954a4e560539f4bf53. at 1732670663818Flushing 0556937df467ac954a4e560539f4bf53/a: creating writer at 1732670663818Flushing 0556937df467ac954a4e560539f4bf53/a: appending metadata at 1732670663842 (+24 ms)Flushing 0556937df467ac954a4e560539f4bf53/a: closing flushed file at 1732670663842Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e233bf: reopening flushed file at 1732670663863 (+21 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 0556937df467ac954a4e560539f4bf53 in 54ms, sequenceid=2, compaction requested=false; wal=null at 1732670663872 (+9 ms)Cleaning up temporary data from old regions at 1732670663874 (+2 ms)Region opened successfully at 1732670663881 (+7 ms) 2024-11-27T01:24:23,904 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=382 (was 372) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57100 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57416 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:51146 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:51410 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:56940 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57160 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=6690 (was 6760) 2024-11-27T01:24:23,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=382, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=6689 2024-11-27T01:24:23,932 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:23,934 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:23,935 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:23,938 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-93023324, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-93023324, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:23,951 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-93023324/hregion-93023324.1732670663938, exclude list is [], retry=0 2024-11-27T01:24:23,955 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:23,955 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:23,956 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:23,958 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-93023324/hregion-93023324.1732670663938 2024-11-27T01:24:23,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:23,958 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 051fa461b549e38d61ce2fc0cd5de043, NAME => 'testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741854_1030 (size=64) 2024-11-27T01:24:23,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741854_1030 (size=64) 2024-11-27T01:24:23,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741854_1030 (size=64) 2024-11-27T01:24:23,969 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:23,970 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,972 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 051fa461b549e38d61ce2fc0cd5de043 columnFamilyName a 2024-11-27T01:24:23,972 DEBUG [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:23,973 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(327): Store=051fa461b549e38d61ce2fc0cd5de043/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:23,973 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,974 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 051fa461b549e38d61ce2fc0cd5de043 columnFamilyName b 2024-11-27T01:24:23,975 DEBUG [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:23,975 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(327): Store=051fa461b549e38d61ce2fc0cd5de043/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:23,975 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,977 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 051fa461b549e38d61ce2fc0cd5de043 columnFamilyName c 2024-11-27T01:24:23,977 DEBUG [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:23,977 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(327): Store=051fa461b549e38d61ce2fc0cd5de043/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:23,977 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,980 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,980 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,981 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:23,982 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:23,985 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:23,985 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 051fa461b549e38d61ce2fc0cd5de043; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62608313, jitterRate=-0.06706343591213226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:23,986 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 051fa461b549e38d61ce2fc0cd5de043: Writing region info on filesystem at 1732670663969Initializing all the Stores at 1732670663970 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670663970Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670663970Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670663970Cleaning up temporary data from old regions at 1732670663980 (+10 ms)Region opened successfully at 1732670663986 (+6 ms) 2024-11-27T01:24:23,986 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 051fa461b549e38d61ce2fc0cd5de043, disabling compactions & flushes 2024-11-27T01:24:23,986 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:23,986 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:23,986 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. after waiting 0 ms 2024-11-27T01:24:23,986 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:23,987 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:23,987 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 051fa461b549e38d61ce2fc0cd5de043: Waiting for close lock at 1732670663986Disabling compacts and flushes for region at 1732670663986Disabling writes for close at 1732670663986Writing region close event to WAL at 1732670663987 (+1 ms)Closed at 1732670663987 2024-11-27T01:24:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741853_1029 (size=95) 2024-11-27T01:24:23,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741853_1029 (size=95) 2024-11-27T01:24:23,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741853_1029 (size=95) 2024-11-27T01:24:23,993 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:23,994 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-93023324:(num 1732670663938) 2024-11-27T01:24:23,994 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:23,996 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:24,010 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996, exclude list is [], retry=0 2024-11-27T01:24:24,013 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:24,013 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:24,014 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:24,016 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 2024-11-27T01:24:24,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867)] 2024-11-27T01:24:24,225 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996, size=0 (0bytes) 2024-11-27T01:24:24,225 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 might be still open, length is 0 2024-11-27T01:24:24,226 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 2024-11-27T01:24:24,227 WARN [IPC Server handler 0 on default port 41069 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-27T01:24:24,227 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 after 1ms 2024-11-27T01:24:24,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57456 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57456 dst: /127.0.0.1:37205 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37205 remote=/127.0.0.1:57456]. Total timeout mills is 60000, 59425 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:24,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:51446 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45195:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51446 dst: /127.0.0.1:45195 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:24,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57134 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36269:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57134 dst: /127.0.0.1:36269 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:24,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741855_1032 (size=263633) 2024-11-27T01:24:24,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741855_1032 (size=263633) 2024-11-27T01:24:24,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741855_1032 (size=263633) 2024-11-27T01:24:28,085 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:24:28,150 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T01:24:28,228 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 after 4002ms 2024-11-27T01:24:28,233 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:28,234 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 took 4009ms 2024-11-27T01:24:28,239 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732670663996.temp 2024-11-27T01:24:28,244 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000000001-wal.1732670663996.temp 2024-11-27T01:24:28,357 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996; continuing. 2024-11-27T01:24:28,357 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 so closing down 2024-11-27T01:24:28,357 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:28,357 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:28,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741856_1033 (size=263641) 2024-11-27T01:24:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741856_1033 (size=263641) 2024-11-27T01:24:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741856_1033 (size=263641) 2024-11-27T01:24:28,362 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000000001-wal.1732670663996.temp (wrote 3002 edits, skipped 0 edits in 75 ms) 2024-11-27T01:24:28,364 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000000001-wal.1732670663996.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000003002 2024-11-27T01:24:28,364 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 129 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:24:28,364 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996, journal: Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996, size=0 (0bytes) at 1732670664225Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000000001-wal.1732670663996.temp at 1732670668244 (+4019 ms)Split 1024 edits, skipped 0 edits. at 1732670668302 (+58 ms)Split 2048 edits, skipped 0 edits. at 1732670668331 (+29 ms)Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 so closing down at 1732670668357 (+26 ms)3 split writer threads finished at 1732670668357Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000000001-wal.1732670663996.temp (wrote 3002 edits, skipped 0 edits in 75 ms) at 1732670668362 (+5 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000000001-wal.1732670663996.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000003002 at 1732670668364 (+2 ms)Processed 3002 edits across 1 Regions in 129 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996, size=0, length=0, corrupted=false, cancelled=false at 1732670668364 2024-11-27T01:24:28,367 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670663996 2024-11-27T01:24:28,368 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000003002 2024-11-27T01:24:28,368 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:28,371 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:28,392 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670668372, exclude list is [], retry=0 2024-11-27T01:24:28,396 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:28,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:28,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:28,400 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670668372 2024-11-27T01:24:28,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:28,401 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:28,403 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:28,404 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 051fa461b549e38d61ce2fc0cd5de043 columnFamilyName a 2024-11-27T01:24:28,404 DEBUG [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:28,405 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(327): Store=051fa461b549e38d61ce2fc0cd5de043/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:28,405 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:28,407 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 051fa461b549e38d61ce2fc0cd5de043 columnFamilyName b 2024-11-27T01:24:28,407 DEBUG [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:28,407 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(327): Store=051fa461b549e38d61ce2fc0cd5de043/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:28,408 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:28,409 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 051fa461b549e38d61ce2fc0cd5de043 columnFamilyName c 2024-11-27T01:24:28,409 DEBUG [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:28,410 INFO [StoreOpener-051fa461b549e38d61ce2fc0cd5de043-1 {}] regionserver.HStore(327): Store=051fa461b549e38d61ce2fc0cd5de043/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:28,410 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:28,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:28,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:28,419 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000003002 2024-11-27T01:24:28,423 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:28,467 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:24:28,864 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 051fa461b549e38d61ce2fc0cd5de043 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T01:24:28,905 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/a/2384aa66b5fc4595a4dcd8d2f3517198 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732670664022/Put/seqid=0 2024-11-27T01:24:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741858_1035 (size=50463) 2024-11-27T01:24:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741858_1035 (size=50463) 2024-11-27T01:24:28,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741858_1035 (size=50463) 2024-11-27T01:24:28,920 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/a/2384aa66b5fc4595a4dcd8d2f3517198 2024-11-27T01:24:28,930 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/a/2384aa66b5fc4595a4dcd8d2f3517198 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/a/2384aa66b5fc4595a4dcd8d2f3517198 2024-11-27T01:24:28,938 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/a/2384aa66b5fc4595a4dcd8d2f3517198, entries=754, sequenceid=754, filesize=49.3 K 2024-11-27T01:24:28,939 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 051fa461b549e38d61ce2fc0cd5de043 in 75ms, sequenceid=754, compaction requested=false; wal=null 2024-11-27T01:24:28,955 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:24:28,955 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 051fa461b549e38d61ce2fc0cd5de043 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T01:24:28,968 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/a/022cf5ac9f974dc49e12005819ad3a7d is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732670664057/Put/seqid=0 2024-11-27T01:24:28,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741859_1036 (size=20072) 2024-11-27T01:24:28,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741859_1036 (size=20072) 2024-11-27T01:24:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741859_1036 (size=20072) 2024-11-27T01:24:28,996 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/a/022cf5ac9f974dc49e12005819ad3a7d 2024-11-27T01:24:29,039 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/b/30ba84c204574158bc62c8130384c0a4 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732670664082/Put/seqid=0 2024-11-27T01:24:29,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741860_1037 (size=35835) 2024-11-27T01:24:29,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741860_1037 (size=35835) 2024-11-27T01:24:29,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741860_1037 (size=35835) 2024-11-27T01:24:29,051 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/b/30ba84c204574158bc62c8130384c0a4 2024-11-27T01:24:29,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/a/022cf5ac9f974dc49e12005819ad3a7d as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/a/022cf5ac9f974dc49e12005819ad3a7d 2024-11-27T01:24:29,066 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/a/022cf5ac9f974dc49e12005819ad3a7d, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-27T01:24:29,068 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/b/30ba84c204574158bc62c8130384c0a4 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/b/30ba84c204574158bc62c8130384c0a4 2024-11-27T01:24:29,075 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/b/30ba84c204574158bc62c8130384c0a4, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-27T01:24:29,075 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 051fa461b549e38d61ce2fc0cd5de043 in 120ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-27T01:24:29,091 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:24:29,091 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 051fa461b549e38d61ce2fc0cd5de043 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T01:24:29,101 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/b/2a346b3f3e304b7eabe2d25f75c0c5cb is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732670664103/Put/seqid=0 2024-11-27T01:24:29,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741861_1038 (size=35082) 2024-11-27T01:24:29,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741861_1038 (size=35082) 2024-11-27T01:24:29,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741861_1038 (size=35082) 2024-11-27T01:24:29,118 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/b/2a346b3f3e304b7eabe2d25f75c0c5cb 2024-11-27T01:24:29,149 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/c/c33d18f8151e455888c5c5566679455e is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732670664134/Put/seqid=0 2024-11-27T01:24:29,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741862_1039 (size=20825) 2024-11-27T01:24:29,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741862_1039 (size=20825) 2024-11-27T01:24:29,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741862_1039 (size=20825) 2024-11-27T01:24:29,166 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/c/c33d18f8151e455888c5c5566679455e 2024-11-27T01:24:29,175 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/b/2a346b3f3e304b7eabe2d25f75c0c5cb as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/b/2a346b3f3e304b7eabe2d25f75c0c5cb 2024-11-27T01:24:29,182 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/b/2a346b3f3e304b7eabe2d25f75c0c5cb, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-27T01:24:29,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/c/c33d18f8151e455888c5c5566679455e as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/c/c33d18f8151e455888c5c5566679455e 2024-11-27T01:24:29,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/c/c33d18f8151e455888c5c5566679455e, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-27T01:24:29,190 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 051fa461b549e38d61ce2fc0cd5de043 in 99ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-27T01:24:29,204 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732670664182/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:29,208 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000003002 2024-11-27T01:24:29,209 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:24:29,209 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 051fa461b549e38d61ce2fc0cd5de043 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-27T01:24:29,217 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/c/3988ebf1144f4db8ace48614c041ec0f is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732670664143/Put/seqid=0 2024-11-27T01:24:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741863_1040 (size=50301) 2024-11-27T01:24:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741863_1040 (size=50301) 2024-11-27T01:24:29,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741863_1040 (size=50301) 2024-11-27T01:24:29,235 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/c/3988ebf1144f4db8ace48614c041ec0f 2024-11-27T01:24:29,242 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3988ebf1144f4db8ace48614c041ec0f 2024-11-27T01:24:29,243 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/.tmp/c/3988ebf1144f4db8ace48614c041ec0f as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/c/3988ebf1144f4db8ace48614c041ec0f 2024-11-27T01:24:29,252 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3988ebf1144f4db8ace48614c041ec0f 2024-11-27T01:24:29,252 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/c/3988ebf1144f4db8ace48614c041ec0f, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-27T01:24:29,252 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 051fa461b549e38d61ce2fc0cd5de043 in 43ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-27T01:24:29,253 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/0000000000000003002 2024-11-27T01:24:29,255 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:29,255 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:29,255 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:24:29,257 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 051fa461b549e38d61ce2fc0cd5de043 2024-11-27T01:24:29,260 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenIntoWAL/051fa461b549e38d61ce2fc0cd5de043/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-27T01:24:29,261 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 051fa461b549e38d61ce2fc0cd5de043; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72047891, jitterRate=0.07359723746776581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:24:29,262 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 051fa461b549e38d61ce2fc0cd5de043: Writing region info on filesystem at 1732670668401Initializing all the Stores at 1732670668402 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670668402Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670668403 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670668403Cleaning up temporary data from old regions at 1732670669255 (+852 ms)Region opened successfully at 1732670669261 (+6 ms) 2024-11-27T01:24:29,324 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 051fa461b549e38d61ce2fc0cd5de043, disabling compactions & flushes 2024-11-27T01:24:29,324 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:29,324 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:29,324 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. after waiting 0 ms 2024-11-27T01:24:29,324 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:29,326 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670663932.051fa461b549e38d61ce2fc0cd5de043. 2024-11-27T01:24:29,326 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 051fa461b549e38d61ce2fc0cd5de043: Waiting for close lock at 1732670669324Disabling compacts and flushes for region at 1732670669324Disabling writes for close at 1732670669324Writing region close event to WAL at 1732670669326 (+2 ms)Closed at 1732670669326 2024-11-27T01:24:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741857_1034 (size=95) 2024-11-27T01:24:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741857_1034 (size=95) 2024-11-27T01:24:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741857_1034 (size=95) 2024-11-27T01:24:29,334 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:29,334 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670668372) 2024-11-27T01:24:29,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=396 (was 382) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43843 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35757 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1713226648_22 at /127.0.0.1:57150 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1713226648_22 at /127.0.0.1:51474 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1713226648_22 at /127.0.0.1:57500 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1774832404) connection to localhost/127.0.0.1:43843 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1774832404) connection to localhost/127.0.0.1:41069 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1713226648_22 at /127.0.0.1:57492 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1774832404) connection to localhost/127.0.0.1:35757 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:41069 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=6531 (was 6689) 2024-11-27T01:24:29,366 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=396, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=6528 2024-11-27T01:24:29,384 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:29,387 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:29,387 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:29,390 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-94725045, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-94725045, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:29,403 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-94725045/hregion-94725045.1732670669390, exclude list is [], retry=0 2024-11-27T01:24:29,406 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:29,407 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:29,407 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:29,410 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-94725045/hregion-94725045.1732670669390 2024-11-27T01:24:29,410 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:29,410 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9d31b023936e2007706f291d7a7fd6bd, NAME => 'test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:29,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741865_1042 (size=43) 2024-11-27T01:24:29,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741865_1042 (size=43) 2024-11-27T01:24:29,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741865_1042 (size=43) 2024-11-27T01:24:29,429 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:29,432 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,433 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d31b023936e2007706f291d7a7fd6bd columnFamilyName a 2024-11-27T01:24:29,433 DEBUG [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:29,434 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(327): Store=9d31b023936e2007706f291d7a7fd6bd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:29,434 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,436 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d31b023936e2007706f291d7a7fd6bd columnFamilyName b 2024-11-27T01:24:29,436 DEBUG [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:29,437 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(327): Store=9d31b023936e2007706f291d7a7fd6bd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:29,437 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,439 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d31b023936e2007706f291d7a7fd6bd columnFamilyName c 2024-11-27T01:24:29,439 DEBUG [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:29,439 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(327): Store=9d31b023936e2007706f291d7a7fd6bd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:29,439 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,440 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,441 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,442 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,442 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,442 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:29,444 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:29,446 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:29,447 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d31b023936e2007706f291d7a7fd6bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72116520, jitterRate=0.07461988925933838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:29,448 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d31b023936e2007706f291d7a7fd6bd: Writing region info on filesystem at 1732670669429Initializing all the Stores at 1732670669430 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670669430Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670669431 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670669431Cleaning up temporary data from old regions at 1732670669442 (+11 ms)Region opened successfully at 1732670669448 (+6 ms) 2024-11-27T01:24:29,448 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9d31b023936e2007706f291d7a7fd6bd, disabling compactions & flushes 2024-11-27T01:24:29,448 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:29,448 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:29,448 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. after waiting 0 ms 2024-11-27T01:24:29,448 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:29,448 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:29,449 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9d31b023936e2007706f291d7a7fd6bd: Waiting for close lock at 1732670669448Disabling compacts and flushes for region at 1732670669448Disabling writes for close at 1732670669448Writing region close event to WAL at 1732670669448Closed at 1732670669448 2024-11-27T01:24:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741864_1041 (size=95) 2024-11-27T01:24:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741864_1041 (size=95) 2024-11-27T01:24:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741864_1041 (size=95) 2024-11-27T01:24:29,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:29,457 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-94725045:(num 1732670669390) 2024-11-27T01:24:29,458 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:29,460 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:29,474 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460, exclude list is [], retry=0 2024-11-27T01:24:29,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:29,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:29,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:29,480 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460 2024-11-27T01:24:29,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:29,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741866_1043 (size=263359) 2024-11-27T01:24:29,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741866_1043 (size=263359) 2024-11-27T01:24:29,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741866_1043 (size=263359) 2024-11-27T01:24:29,700 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460, size=257.2 K (263359bytes) 2024-11-27T01:24:29,700 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460 2024-11-27T01:24:29,701 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460 after 0ms 2024-11-27T01:24:29,704 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:29,705 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460 took 5ms 2024-11-27T01:24:29,717 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732670669460.temp 2024-11-27T01:24:29,719 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000000001-wal.1732670669460.temp 2024-11-27T01:24:29,780 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460 so closing down 2024-11-27T01:24:29,780 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:29,780 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:29,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741867_1044 (size=263359) 2024-11-27T01:24:29,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741867_1044 (size=263359) 2024-11-27T01:24:29,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741867_1044 (size=263359) 2024-11-27T01:24:29,787 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000000001-wal.1732670669460.temp (wrote 3000 edits, skipped 0 edits in 45 ms) 2024-11-27T01:24:29,789 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000000001-wal.1732670669460.temp to hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003000 2024-11-27T01:24:29,789 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 83 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-27T01:24:29,789 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460, journal: Splitting hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460, size=257.2 K (263359bytes) at 1732670669700Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000000001-wal.1732670669460.temp at 1732670669719 (+19 ms)Split 1024 edits, skipped 0 edits. at 1732670669731 (+12 ms)Split 2048 edits, skipped 0 edits. at 1732670669756 (+25 ms)Finishing writing output for hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460 so closing down at 1732670669780 (+24 ms)3 split writer threads finished at 1732670669780Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000000001-wal.1732670669460.temp (wrote 3000 edits, skipped 0 edits in 45 ms) at 1732670669787 (+7 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000000001-wal.1732670669460.temp to hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003000 at 1732670669789 (+2 ms)Processed 3000 edits across 1 Regions in 83 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1732670669789 2024-11-27T01:24:29,791 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669460 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670669460 2024-11-27T01:24:29,792 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003000 2024-11-27T01:24:29,793 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:29,795 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:29,810 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795, exclude list is [], retry=0 2024-11-27T01:24:29,813 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:29,814 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:29,814 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:29,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795 2024-11-27T01:24:29,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:30,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741868_1045 (size=263486) 2024-11-27T01:24:30,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741868_1045 (size=263486) 2024-11-27T01:24:30,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741868_1045 (size=263486) 2024-11-27T01:24:30,047 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795, size=257.3 K (263486bytes) 2024-11-27T01:24:30,047 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795 2024-11-27T01:24:30,047 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795 after 0ms 2024-11-27T01:24:30,050 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:30,052 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795 took 6ms 2024-11-27T01:24:30,057 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732670669795.temp 2024-11-27T01:24:30,059 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003001-wal.1732670669795.temp 2024-11-27T01:24:30,122 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795 so closing down 2024-11-27T01:24:30,122 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:30,122 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:30,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741869_1046 (size=263486) 2024-11-27T01:24:30,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741869_1046 (size=263486) 2024-11-27T01:24:30,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741869_1046 (size=263486) 2024-11-27T01:24:30,127 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003001-wal.1732670669795.temp (wrote 3000 edits, skipped 0 edits in 39 ms) 2024-11-27T01:24:30,129 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003001-wal.1732670669795.temp to hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000006000 2024-11-27T01:24:30,129 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-27T01:24:30,129 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795, journal: Splitting hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795, size=257.3 K (263486bytes) at 1732670670047Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003001-wal.1732670669795.temp at 1732670670059 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732670670078 (+19 ms)Split 2048 edits, skipped 0 edits. at 1732670670100 (+22 ms)Finishing writing output for hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795 so closing down at 1732670670122 (+22 ms)3 split writer threads finished at 1732670670122Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003001-wal.1732670669795.temp (wrote 3000 edits, skipped 0 edits in 39 ms) at 1732670670127 (+5 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003001-wal.1732670669795.temp to hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000006000 at 1732670670129 (+2 ms)Processed 3000 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1732670670129 2024-11-27T01:24:30,131 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670669795 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670669795 2024-11-27T01:24:30,132 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000006000 2024-11-27T01:24:30,132 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:30,134 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/test2727-manual,16010,1732670669383, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:30,187 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670670135, exclude list is [], retry=0 2024-11-27T01:24:30,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:30,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:30,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:30,212 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732670669383/wal.1732670670135 2024-11-27T01:24:30,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:30,213 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d31b023936e2007706f291d7a7fd6bd, NAME => 'test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:30,213 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:30,213 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,214 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,217 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,218 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d31b023936e2007706f291d7a7fd6bd columnFamilyName a 2024-11-27T01:24:30,218 DEBUG [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:30,219 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(327): Store=9d31b023936e2007706f291d7a7fd6bd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:30,219 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,220 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d31b023936e2007706f291d7a7fd6bd columnFamilyName b 2024-11-27T01:24:30,220 DEBUG [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:30,226 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(327): Store=9d31b023936e2007706f291d7a7fd6bd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:30,227 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,228 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d31b023936e2007706f291d7a7fd6bd columnFamilyName c 2024-11-27T01:24:30,228 DEBUG [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:30,229 INFO [StoreOpener-9d31b023936e2007706f291d7a7fd6bd-1 {}] regionserver.HStore(327): Store=9d31b023936e2007706f291d7a7fd6bd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:30,229 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,230 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,232 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,233 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003000 2024-11-27T01:24:30,237 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:30,290 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003000 2024-11-27T01:24:30,292 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000006000 2024-11-27T01:24:30,295 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:30,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T01:24:30,299 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:30,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:24:30,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T01:24:30,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T01:24:30,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:30,302 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-27T01:24:30,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:30,343 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000006000 2024-11-27T01:24:30,343 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d31b023936e2007706f291d7a7fd6bd 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-27T01:24:30,375 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/a/56b08c3a568d4a5c88b25748bdc961dc is 41, key is test2727/a:100/1732670669822/Put/seqid=0 2024-11-27T01:24:30,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741871_1048 (size=84227) 2024-11-27T01:24:30,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741871_1048 (size=84227) 2024-11-27T01:24:30,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741871_1048 (size=84227) 2024-11-27T01:24:30,384 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/a/56b08c3a568d4a5c88b25748bdc961dc 2024-11-27T01:24:30,415 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/b/b59e95cd808e425da1799395c23ce512 is 41, key is test2727/b:100/1732670669890/Put/seqid=0 2024-11-27T01:24:30,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741872_1049 (size=84609) 2024-11-27T01:24:30,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741872_1049 (size=84609) 2024-11-27T01:24:30,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741872_1049 (size=84609) 2024-11-27T01:24:30,421 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/b/b59e95cd808e425da1799395c23ce512 2024-11-27T01:24:30,447 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/c/73412b0d850d4db1a58c96ebd01f0ce4 is 41, key is test2727/c:100/1732670669955/Put/seqid=0 2024-11-27T01:24:30,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741873_1050 (size=84609) 2024-11-27T01:24:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741873_1050 (size=84609) 2024-11-27T01:24:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741873_1050 (size=84609) 2024-11-27T01:24:30,858 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/c/73412b0d850d4db1a58c96ebd01f0ce4 2024-11-27T01:24:30,865 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/a/56b08c3a568d4a5c88b25748bdc961dc as hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/a/56b08c3a568d4a5c88b25748bdc961dc 2024-11-27T01:24:30,870 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/a/56b08c3a568d4a5c88b25748bdc961dc, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-27T01:24:30,871 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/b/b59e95cd808e425da1799395c23ce512 as hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/b/b59e95cd808e425da1799395c23ce512 2024-11-27T01:24:30,876 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/b/b59e95cd808e425da1799395c23ce512, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T01:24:30,877 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/.tmp/c/73412b0d850d4db1a58c96ebd01f0ce4 as hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/c/73412b0d850d4db1a58c96ebd01f0ce4 2024-11-27T01:24:30,882 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/c/73412b0d850d4db1a58c96ebd01f0ce4, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T01:24:30,883 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 9d31b023936e2007706f291d7a7fd6bd in 540ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-27T01:24:30,883 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000003000 2024-11-27T01:24:30,884 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/0000000000000006000 2024-11-27T01:24:30,885 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,885 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,886 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:30,887 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d31b023936e2007706f291d7a7fd6bd 2024-11-27T01:24:30,890 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/test2727/9d31b023936e2007706f291d7a7fd6bd/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-27T01:24:30,891 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d31b023936e2007706f291d7a7fd6bd; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73947057, jitterRate=0.10189701616764069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:30,891 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d31b023936e2007706f291d7a7fd6bd: Writing region info on filesystem at 1732670670214Initializing all the Stores at 1732670670215 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670670215Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670670216 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670670216Obtaining lock to block concurrent updates at 1732670670343 (+127 ms)Preparing flush snapshotting stores in 9d31b023936e2007706f291d7a7fd6bd at 1732670670343Finished memstore snapshotting test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732670670344 (+1 ms)Flushing stores of test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. at 1732670670344Flushing 9d31b023936e2007706f291d7a7fd6bd/a: creating writer at 1732670670344Flushing 9d31b023936e2007706f291d7a7fd6bd/a: appending metadata at 1732670670374 (+30 ms)Flushing 9d31b023936e2007706f291d7a7fd6bd/a: closing flushed file at 1732670670374Flushing 9d31b023936e2007706f291d7a7fd6bd/b: creating writer at 1732670670390 (+16 ms)Flushing 9d31b023936e2007706f291d7a7fd6bd/b: appending metadata at 1732670670413 (+23 ms)Flushing 9d31b023936e2007706f291d7a7fd6bd/b: closing flushed file at 1732670670413Flushing 9d31b023936e2007706f291d7a7fd6bd/c: creating writer at 1732670670427 (+14 ms)Flushing 9d31b023936e2007706f291d7a7fd6bd/c: appending metadata at 1732670670446 (+19 ms)Flushing 9d31b023936e2007706f291d7a7fd6bd/c: closing flushed file at 1732670670446Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32341f8c: reopening flushed file at 1732670670864 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1973e29f: reopening flushed file at 1732670670870 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2920dd76: reopening flushed file at 1732670670876 (+6 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 9d31b023936e2007706f291d7a7fd6bd in 540ms, sequenceid=6000, compaction requested=false; wal=null at 1732670670883 (+7 ms)Cleaning up temporary data from old regions at 1732670670885 (+2 ms)Region opened successfully at 1732670670891 (+6 ms) 2024-11-27T01:24:30,893 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-27T01:24:30,893 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9d31b023936e2007706f291d7a7fd6bd, disabling compactions & flushes 2024-11-27T01:24:30,893 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:30,893 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:30,893 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. after waiting 0 ms 2024-11-27T01:24:30,893 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:30,894 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732670669385.9d31b023936e2007706f291d7a7fd6bd. 2024-11-27T01:24:30,894 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9d31b023936e2007706f291d7a7fd6bd: Waiting for close lock at 1732670670893Disabling compacts and flushes for region at 1732670670893Disabling writes for close at 1732670670893Writing region close event to WAL at 1732670670894 (+1 ms)Closed at 1732670670894 2024-11-27T01:24:30,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741870_1047 (size=95) 2024-11-27T01:24:30,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741870_1047 (size=95) 2024-11-27T01:24:30,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741870_1047 (size=95) 2024-11-27T01:24:30,899 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:30,899 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670670135) 2024-11-27T01:24:30,915 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=398 (was 396) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57500 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:51632 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57278 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=245 (was 231) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6306 (was 6528) 2024-11-27T01:24:30,927 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=398, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=245, ProcessCount=11, AvailableMemoryMB=6306 2024-11-27T01:24:30,942 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:30,949 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:30,949 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732670670949 2024-11-27T01:24:30,956 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949 2024-11-27T01:24:30,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:30,959 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f9affbf67e1579dc1fa6f361a1fc9c44, NAME => 'testSequentialEditLogSeqNum,,1732670670943.f9affbf67e1579dc1fa6f361a1fc9c44.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:30,960 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732670670943.f9affbf67e1579dc1fa6f361a1fc9c44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:30,960 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,960 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,961 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44 doesn't exist for region: f9affbf67e1579dc1fa6f361a1fc9c44 on table testSequentialEditLogSeqNum 2024-11-27T01:24:30,961 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: f9affbf67e1579dc1fa6f361a1fc9c44 on table testSequentialEditLogSeqNum 2024-11-27T01:24:30,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741875_1052 (size=62) 2024-11-27T01:24:30,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741875_1052 (size=62) 2024-11-27T01:24:30,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741875_1052 (size=62) 2024-11-27T01:24:30,971 INFO [StoreOpener-f9affbf67e1579dc1fa6f361a1fc9c44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,972 INFO [StoreOpener-f9affbf67e1579dc1fa6f361a1fc9c44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9affbf67e1579dc1fa6f361a1fc9c44 columnFamilyName a 2024-11-27T01:24:30,972 DEBUG [StoreOpener-f9affbf67e1579dc1fa6f361a1fc9c44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:30,973 INFO [StoreOpener-f9affbf67e1579dc1fa6f361a1fc9c44-1 {}] regionserver.HStore(327): Store=f9affbf67e1579dc1fa6f361a1fc9c44/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:30,973 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,974 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,974 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,974 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,976 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f9affbf67e1579dc1fa6f361a1fc9c44 2024-11-27T01:24:30,978 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:30,978 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f9affbf67e1579dc1fa6f361a1fc9c44; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75000570, jitterRate=0.11759558320045471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:30,979 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f9affbf67e1579dc1fa6f361a1fc9c44: Writing region info on filesystem at 1732670670960Initializing all the Stores at 1732670670970 (+10 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670670970Cleaning up temporary data from old regions at 1732670670974 (+4 ms)Region opened successfully at 1732670670979 (+5 ms) 2024-11-27T01:24:30,990 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f9affbf67e1579dc1fa6f361a1fc9c44 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-27T01:24:31,012 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/.tmp/a/76900eb0193a42e6a4a23c229fc89ee3 is 81, key is testSequentialEditLogSeqNum/a:x0/1732670670979/Put/seqid=0 2024-11-27T01:24:31,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741876_1053 (size=5833) 2024-11-27T01:24:31,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741876_1053 (size=5833) 2024-11-27T01:24:31,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741876_1053 (size=5833) 2024-11-27T01:24:31,020 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/.tmp/a/76900eb0193a42e6a4a23c229fc89ee3 2024-11-27T01:24:31,026 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/.tmp/a/76900eb0193a42e6a4a23c229fc89ee3 as hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/a/76900eb0193a42e6a4a23c229fc89ee3 2024-11-27T01:24:31,032 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/a/76900eb0193a42e6a4a23c229fc89ee3, entries=10, sequenceid=13, filesize=5.7 K 2024-11-27T01:24:31,034 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f9affbf67e1579dc1fa6f361a1fc9c44 in 43ms, sequenceid=13, compaction requested=false 2024-11-27T01:24:31,034 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f9affbf67e1579dc1fa6f361a1fc9c44: 2024-11-27T01:24:31,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:24:31,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:24:31,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:24:31,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:24:31,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:24:31,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741874_1051 (size=1843) 2024-11-27T01:24:31,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741874_1051 (size=1843) 2024-11-27T01:24:31,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741874_1051 (size=1843) 2024-11-27T01:24:31,058 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949, size=1.8 K (1843bytes) 2024-11-27T01:24:31,058 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949 2024-11-27T01:24:31,058 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949 after 0ms 2024-11-27T01:24:31,061 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:31,061 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949 took 3ms 2024-11-27T01:24:31,064 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949 so closing down 2024-11-27T01:24:31,064 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:31,065 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670670949.temp 2024-11-27T01:24:31,066 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000003-wal.1732670670949.temp 2024-11-27T01:24:31,067 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:31,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741877_1054 (size=1477) 2024-11-27T01:24:31,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741877_1054 (size=1477) 2024-11-27T01:24:31,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741877_1054 (size=1477) 2024-11-27T01:24:31,073 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000003-wal.1732670670949.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:31,075 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000003-wal.1732670670949.temp to hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000020 2024-11-27T01:24:31,075 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949, size=1.8 K, length=1843, corrupted=false, cancelled=false 2024-11-27T01:24:31,075 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949, journal: Splitting hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949, size=1.8 K (1843bytes) at 1732670671058Finishing writing output for hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949 so closing down at 1732670671064 (+6 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000003-wal.1732670670949.temp at 1732670671066 (+2 ms)3 split writer threads finished at 1732670671067 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000003-wal.1732670670949.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732670671073 (+6 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000003-wal.1732670670949.temp to hdfs://localhost:41069/hbase/data/default/testSequentialEditLogSeqNum/f9affbf67e1579dc1fa6f361a1fc9c44/recovered.edits/0000000000000000020 at 1732670671075 (+2 ms)Processed 17 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:41069/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670670942/wal.1732670670949, size=1.8 K, length=1843, corrupted=false, cancelled=false at 1732670671075 2024-11-27T01:24:31,090 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=403 (was 398) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57500 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:51632 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=245 (was 245), ProcessCount=11 (was 11), AvailableMemoryMB=6299 (was 6306) 2024-11-27T01:24:31,102 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=403, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=245, ProcessCount=11, AvailableMemoryMB=6298 2024-11-27T01:24:31,120 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:31,123 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:31,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:31,160 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97665719, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-97665719, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:31,174 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97665719/hregion-97665719.1732670671161, exclude list is [], retry=0 2024-11-27T01:24:31,177 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:31,178 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:31,179 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:31,183 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97665719/hregion-97665719.1732670671161 2024-11-27T01:24:31,183 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867)] 2024-11-27T01:24:31,183 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1b104b69ed21032cdd60062976e31f39, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:31,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741879_1056 (size=70) 2024-11-27T01:24:31,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741879_1056 (size=70) 2024-11-27T01:24:31,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741879_1056 (size=70) 2024-11-27T01:24:31,198 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:31,199 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,200 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName a 2024-11-27T01:24:31,201 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:31,201 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:31,201 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,203 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName b 2024-11-27T01:24:31,203 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:31,203 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:31,203 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,204 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName c 2024-11-27T01:24:31,204 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:31,205 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:31,205 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,207 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,207 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,207 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:31,209 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,213 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:31,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b104b69ed21032cdd60062976e31f39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61762546, jitterRate=-0.07966634631156921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:31,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b104b69ed21032cdd60062976e31f39: Writing region info on filesystem at 1732670671198Initializing all the Stores at 1732670671199 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670671199Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670671199Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670671199Cleaning up temporary data from old regions at 1732670671207 (+8 ms)Region opened successfully at 1732670671214 (+7 ms) 2024-11-27T01:24:31,214 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1b104b69ed21032cdd60062976e31f39, disabling compactions & flushes 2024-11-27T01:24:31,214 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:31,214 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:31,214 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. after waiting 0 ms 2024-11-27T01:24:31,214 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:31,215 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:31,215 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1b104b69ed21032cdd60062976e31f39: Waiting for close lock at 1732670671214Disabling compacts and flushes for region at 1732670671214Disabling writes for close at 1732670671214Writing region close event to WAL at 1732670671215 (+1 ms)Closed at 1732670671215 2024-11-27T01:24:31,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741878_1055 (size=95) 2024-11-27T01:24:31,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741878_1055 (size=95) 2024-11-27T01:24:31,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741878_1055 (size=95) 2024-11-27T01:24:31,221 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:31,221 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97665719:(num 1732670671161) 2024-11-27T01:24:31,221 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:31,223 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:31,237 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224, exclude list is [], retry=0 2024-11-27T01:24:31,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:31,241 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:31,242 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:31,243 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 2024-11-27T01:24:31,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:31,244 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b104b69ed21032cdd60062976e31f39, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:31,244 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:31,244 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,244 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,246 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,247 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName a 2024-11-27T01:24:31,247 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:31,248 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:31,248 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,249 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName b 2024-11-27T01:24:31,249 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:31,249 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:31,249 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,250 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName c 2024-11-27T01:24:31,250 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:31,251 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:31,251 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,252 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,253 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,254 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,254 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,254 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:31,256 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:31,257 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b104b69ed21032cdd60062976e31f39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65106366, jitterRate=-0.029839545488357544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:31,258 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b104b69ed21032cdd60062976e31f39: Writing region info on filesystem at 1732670671245Initializing all the Stores at 1732670671246 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670671246Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670671246Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670671246Cleaning up temporary data from old regions at 1732670671254 (+8 ms)Region opened successfully at 1732670671257 (+3 ms) 2024-11-27T01:24:31,262 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732670671261/Put/seqid=0 2024-11-27T01:24:31,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741881_1058 (size=4826) 2024-11-27T01:24:31,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741881_1058 (size=4826) 2024-11-27T01:24:31,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741881_1058 (size=4826) 2024-11-27T01:24:31,270 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41069/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 1b104b69ed21032cdd60062976e31f39/a 2024-11-27T01:24:31,277 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-27T01:24:31,278 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:24:31,278 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1b104b69ed21032cdd60062976e31f39: 2024-11-27T01:24:31,279 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/a/09b2f4ef242a4415bba5bec9273dac4e_SeqId_3_ 2024-11-27T01:24:31,280 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41069/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1b104b69ed21032cdd60062976e31f39/a as hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/a/09b2f4ef242a4415bba5bec9273dac4e_SeqId_3_ - updating store file list. 2024-11-27T01:24:31,285 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 09b2f4ef242a4415bba5bec9273dac4e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:31,285 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/a/09b2f4ef242a4415bba5bec9273dac4e_SeqId_3_ into 1b104b69ed21032cdd60062976e31f39/a 2024-11-27T01:24:31,285 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41069/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1b104b69ed21032cdd60062976e31f39/a (new location: hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/a/09b2f4ef242a4415bba5bec9273dac4e_SeqId_3_) 2024-11-27T01:24:31,331 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224, size=0 (0bytes) 2024-11-27T01:24:31,332 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 might be still open, length is 0 2024-11-27T01:24:31,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 2024-11-27T01:24:31,332 WARN [IPC Server handler 3 on default port 41069 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-27T01:24:31,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 after 0ms 2024-11-27T01:24:33,753 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:37542 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36269:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37542 dst: /127.0.0.1:36269 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36269 remote=/127.0.0.1:37542]. Total timeout mills is 60000, 57540 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:33,754 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57216 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57216 dst: /127.0.0.1:37205 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:33,754 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:49018 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45195:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49018 dst: /127.0.0.1:45195 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:33,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741880_1059 (size=473) 2024-11-27T01:24:33,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741880_1059 (size=473) 2024-11-27T01:24:33,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741880_1059 (size=473) 2024-11-27T01:24:35,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 after 4001ms 2024-11-27T01:24:35,337 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:35,337 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 took 4006ms 2024-11-27T01:24:35,339 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224; continuing. 2024-11-27T01:24:35,339 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 so closing down 2024-11-27T01:24:35,339 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:35,341 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732670671224.temp 2024-11-27T01:24:35,342 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005-wal.1732670671224.temp 2024-11-27T01:24:35,343 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:35,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741882_1060 (size=259) 2024-11-27T01:24:35,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741882_1060 (size=259) 2024-11-27T01:24:35,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741882_1060 (size=259) 2024-11-27T01:24:35,350 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005-wal.1732670671224.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:35,351 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005-wal.1732670671224.temp to hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005 2024-11-27T01:24:35,351 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:24:35,351 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224, journal: Splitting hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224, size=0 (0bytes) at 1732670671332Finishing writing output for hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 so closing down at 1732670675339 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005-wal.1732670671224.temp at 1732670675342 (+3 ms)3 split writer threads finished at 1732670675343 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005-wal.1732670671224.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732670675350 (+7 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005-wal.1732670671224.temp to hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005 at 1732670675351 (+1 ms)Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224, size=0, length=0, corrupted=false, cancelled=false at 1732670675351 2024-11-27T01:24:35,353 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670671224 2024-11-27T01:24:35,354 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005 2024-11-27T01:24:35,354 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:35,355 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:35,369 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670675356, exclude list is [], retry=0 2024-11-27T01:24:35,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:35,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:35,372 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:35,374 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670675356 2024-11-27T01:24:35,374 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:35,374 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b104b69ed21032cdd60062976e31f39, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:35,374 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:35,374 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,374 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,376 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,377 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName a 2024-11-27T01:24:35,377 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:35,383 DEBUG [StoreFileOpener-1b104b69ed21032cdd60062976e31f39-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 09b2f4ef242a4415bba5bec9273dac4e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:35,383 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/a/09b2f4ef242a4415bba5bec9273dac4e_SeqId_3_ 2024-11-27T01:24:35,383 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:35,383 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,384 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName b 2024-11-27T01:24:35,384 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:35,385 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:35,385 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,385 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b104b69ed21032cdd60062976e31f39 columnFamilyName c 2024-11-27T01:24:35,386 DEBUG [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:35,386 INFO [StoreOpener-1b104b69ed21032cdd60062976e31f39-1 {}] regionserver.HStore(327): Store=1b104b69ed21032cdd60062976e31f39/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:35,386 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,389 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,390 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005 2024-11-27T01:24:35,392 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:35,393 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005 2024-11-27T01:24:35,393 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1b104b69ed21032cdd60062976e31f39 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-27T01:24:35,409 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/.tmp/a/e12f5191616c4b32a1792acca343f3db is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732670671292/Put/seqid=0 2024-11-27T01:24:35,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741884_1062 (size=5149) 2024-11-27T01:24:35,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741884_1062 (size=5149) 2024-11-27T01:24:35,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741884_1062 (size=5149) 2024-11-27T01:24:35,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/.tmp/a/e12f5191616c4b32a1792acca343f3db 2024-11-27T01:24:35,423 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/.tmp/a/e12f5191616c4b32a1792acca343f3db as hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/a/e12f5191616c4b32a1792acca343f3db 2024-11-27T01:24:35,429 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/a/e12f5191616c4b32a1792acca343f3db, entries=1, sequenceid=5, filesize=5.0 K 2024-11-27T01:24:35,429 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1b104b69ed21032cdd60062976e31f39 in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-11-27T01:24:35,430 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/0000000000000000005 2024-11-27T01:24:35,431 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,431 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,432 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:35,434 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b104b69ed21032cdd60062976e31f39 2024-11-27T01:24:35,436 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1b104b69ed21032cdd60062976e31f39/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-27T01:24:35,437 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b104b69ed21032cdd60062976e31f39; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63489769, jitterRate=-0.053928717970848083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:35,437 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b104b69ed21032cdd60062976e31f39: Writing region info on filesystem at 1732670675375Initializing all the Stores at 1732670675375Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670675375Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670675375Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670675375Obtaining lock to block concurrent updates at 1732670675393 (+18 ms)Preparing flush snapshotting stores in 1b104b69ed21032cdd60062976e31f39 at 1732670675393Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732670675393Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. at 1732670675393Flushing 1b104b69ed21032cdd60062976e31f39/a: creating writer at 1732670675393Flushing 1b104b69ed21032cdd60062976e31f39/a: appending metadata at 1732670675408 (+15 ms)Flushing 1b104b69ed21032cdd60062976e31f39/a: closing flushed file at 1732670675408Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@215d6e75: reopening flushed file at 1732670675422 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1b104b69ed21032cdd60062976e31f39 in 36ms, sequenceid=5, compaction requested=false; wal=null at 1732670675429 (+7 ms)Cleaning up temporary data from old regions at 1732670675431 (+2 ms)Region opened successfully at 1732670675437 (+6 ms) 2024-11-27T01:24:35,442 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1b104b69ed21032cdd60062976e31f39, disabling compactions & flushes 2024-11-27T01:24:35,442 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:35,442 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:35,442 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. after waiting 0 ms 2024-11-27T01:24:35,442 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:35,443 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732670671121.1b104b69ed21032cdd60062976e31f39. 2024-11-27T01:24:35,443 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1b104b69ed21032cdd60062976e31f39: Waiting for close lock at 1732670675442Disabling compacts and flushes for region at 1732670675442Disabling writes for close at 1732670675442Writing region close event to WAL at 1732670675443 (+1 ms)Closed at 1732670675443 2024-11-27T01:24:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741883_1061 (size=95) 2024-11-27T01:24:35,447 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670675356 not finished, retry = 0 2024-11-27T01:24:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741883_1061 (size=95) 2024-11-27T01:24:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741883_1061 (size=95) 2024-11-27T01:24:35,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:35,550 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670675356) 2024-11-27T01:24:35,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=405 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-824289349_22 at /127.0.0.1:57262 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:41069 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1774832404) connection to localhost/127.0.0.1:41069 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-824289349_22 at /127.0.0.1:37560 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 245), ProcessCount=11 (was 11), AvailableMemoryMB=6276 (was 6298) 2024-11-27T01:24:35,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=405, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=6274 2024-11-27T01:24:35,603 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:35,610 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:24:35,615 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:35,618 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b0c1f5f 2024-11-27T01:24:35,619 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:24:35,621 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:24:35,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:24:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T01:24:35,636 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:24:35,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-27T01:24:35,638 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:35,640 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:24:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:24:35,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741885_1063 (size=694) 2024-11-27T01:24:35,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741885_1063 (size=694) 2024-11-27T01:24:35,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741885_1063 (size=694) 2024-11-27T01:24:35,657 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 159a502e808ebaf9e21757c52ae64154, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73 2024-11-27T01:24:35,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741886_1064 (size=77) 2024-11-27T01:24:35,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741886_1064 (size=77) 2024-11-27T01:24:35,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741886_1064 (size=77) 2024-11-27T01:24:35,667 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:35,668 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 159a502e808ebaf9e21757c52ae64154, disabling compactions & flushes 2024-11-27T01:24:35,668 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:35,668 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:35,668 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. after waiting 0 ms 2024-11-27T01:24:35,668 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:35,668 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:35,668 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 159a502e808ebaf9e21757c52ae64154: Waiting for close lock at 1732670675668Disabling compacts and flushes for region at 1732670675668Disabling writes for close at 1732670675668Writing region close event to WAL at 1732670675668Closed at 1732670675668 2024-11-27T01:24:35,670 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:24:35,676 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732670675670"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670675670"}]},"ts":"1732670675670"} 2024-11-27T01:24:35,680 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:24:35,682 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:24:35,685 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732670675682"}]},"ts":"1732670675682"} 2024-11-27T01:24:35,691 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-27T01:24:35,692 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:24:35,694 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:24:35,694 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:24:35,694 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T01:24:35,694 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:24:35,694 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:24:35,694 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T01:24:35,694 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:24:35,694 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:24:35,694 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T01:24:35,694 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:24:35,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN}] 2024-11-27T01:24:35,697 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN 2024-11-27T01:24:35,698 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN; state=OFFLINE, location=7d6f26a4c136,42983,1732670660687; forceNewPlan=false, retain=false 2024-11-27T01:24:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:24:35,851 INFO [7d6f26a4c136:43429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T01:24:35,852 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPENING, regionLocation=7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:35,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN because future has completed 2024-11-27T01:24:35,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687}] 2024-11-27T01:24:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:24:36,009 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:24:36,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51467, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:24:36,016 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,017 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 159a502e808ebaf9e21757c52ae64154, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:36,017 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,017 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:36,017 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,017 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,019 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,021 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf1 2024-11-27T01:24:36,021 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:36,021 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:36,021 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,023 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf2 2024-11-27T01:24:36,023 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:36,024 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:36,024 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,025 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,025 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,026 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,026 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,027 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:24:36,028 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,030 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:36,031 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 159a502e808ebaf9e21757c52ae64154; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73179875, jitterRate=0.09046511352062225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:24:36,031 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,032 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 159a502e808ebaf9e21757c52ae64154: Running coprocessor pre-open hook at 1732670676018Writing region info on filesystem at 1732670676018Initializing all the Stores at 1732670676019 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670676019Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670676019Cleaning up temporary data from old regions at 1732670676026 (+7 ms)Running coprocessor post-open hooks at 1732670676031 (+5 ms)Region opened successfully at 1732670676032 (+1 ms) 2024-11-27T01:24:36,034 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., pid=6, masterSystemTime=1732670676009 2024-11-27T01:24:36,037 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,037 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,038 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:36,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687 because future has completed 2024-11-27T01:24:36,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T01:24:36,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687 in 187 msec 2024-11-27T01:24:36,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T01:24:36,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN in 351 msec 2024-11-27T01:24:36,050 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:24:36,050 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732670676050"}]},"ts":"1732670676050"} 2024-11-27T01:24:36,053 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-27T01:24:36,054 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:24:36,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 426 msec 2024-11-27T01:24:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:24:36,273 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-27T01:24:36,273 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-27T01:24:36,275 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:24:36,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-27T01:24:36,281 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:24:36,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-27T01:24:36,293 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=2] 2024-11-27T01:24:36,294 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:24:36,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:24:36,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=159a502e808ebaf9e21757c52ae64154, source=7d6f26a4c136,42983,1732670660687, destination=7d6f26a4c136,34199,1732670660585, warming up region on 7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:24:36,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=159a502e808ebaf9e21757c52ae64154, source=7d6f26a4c136,42983,1732670660687, destination=7d6f26a4c136,34199,1732670660585, running balancer 2024-11-27T01:24:36,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33531, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:24:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE 2024-11-27T01:24:36,317 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE 2024-11-27T01:24:36,320 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=CLOSING, regionLocation=7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:36,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(7855): Warmup {ENCODED => 159a502e808ebaf9e21757c52ae64154, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:36,323 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE because future has completed 2024-11-27T01:24:36,324 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:24:36,324 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf1 2024-11-27T01:24:36,324 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:36,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687}] 2024-11-27T01:24:36,325 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:36,325 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,326 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf2 2024-11-27T01:24:36,326 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:36,327 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(1722): Closing 159a502e808ebaf9e21757c52ae64154, disabling compactions & flushes 2024-11-27T01:24:36,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. after waiting 0 ms 2024-11-27T01:24:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] regionserver.HRegion(1676): Region close journal for 159a502e808ebaf9e21757c52ae64154: Waiting for close lock at 1732670676327Disabling compacts and flushes for region at 1732670676327Disabling writes for close at 1732670676327Writing region close event to WAL at 1732670676339 (+12 ms)Closed at 1732670676339 2024-11-27T01:24:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-27T01:24:36,485 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,485 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:24:36,486 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 159a502e808ebaf9e21757c52ae64154, disabling compactions & flushes 2024-11-27T01:24:36,486 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,486 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,486 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. after waiting 0 ms 2024-11-27T01:24:36,486 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,486 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 159a502e808ebaf9e21757c52ae64154 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-27T01:24:36,504 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf1/3cfdbb79da6042d09cf16ca0b6a24039 is 35, key is r1/cf1:q/1732670676297/Put/seqid=0 2024-11-27T01:24:36,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741887_1065 (size=4783) 2024-11-27T01:24:36,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741887_1065 (size=4783) 2024-11-27T01:24:36,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741887_1065 (size=4783) 2024-11-27T01:24:36,512 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf1/3cfdbb79da6042d09cf16ca0b6a24039 2024-11-27T01:24:36,520 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf1/3cfdbb79da6042d09cf16ca0b6a24039 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039 2024-11-27T01:24:36,526 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039, entries=1, sequenceid=5, filesize=4.7 K 2024-11-27T01:24:36,527 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 159a502e808ebaf9e21757c52ae64154 in 41ms, sequenceid=5, compaction requested=false 2024-11-27T01:24:36,527 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-27T01:24:36,532 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T01:24:36,535 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,535 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 159a502e808ebaf9e21757c52ae64154: Waiting for close lock at 1732670676486Running coprocessor pre-close hooks at 1732670676486Disabling compacts and flushes for region at 1732670676486Disabling writes for close at 1732670676486Obtaining lock to block concurrent updates at 1732670676486Preparing flush snapshotting stores in 159a502e808ebaf9e21757c52ae64154 at 1732670676486Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732670676487 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. at 1732670676487Flushing 159a502e808ebaf9e21757c52ae64154/cf1: creating writer at 1732670676487Flushing 159a502e808ebaf9e21757c52ae64154/cf1: appending metadata at 1732670676504 (+17 ms)Flushing 159a502e808ebaf9e21757c52ae64154/cf1: closing flushed file at 1732670676504Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e580b6f: reopening flushed file at 1732670676519 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 159a502e808ebaf9e21757c52ae64154 in 41ms, sequenceid=5, compaction requested=false at 1732670676527 (+8 ms)Writing region close event to WAL at 1732670676529 (+2 ms)Running coprocessor post-close hooks at 1732670676533 (+4 ms)Closed at 1732670676535 (+2 ms) 2024-11-27T01:24:36,536 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 159a502e808ebaf9e21757c52ae64154 move to 7d6f26a4c136,34199,1732670660585 record at close sequenceid=5 2024-11-27T01:24:36,540 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,540 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=CLOSED 2024-11-27T01:24:36,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687 because future has completed 2024-11-27T01:24:36,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T01:24:36,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687 in 221 msec 2024-11-27T01:24:36,550 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE; state=CLOSED, location=7d6f26a4c136,34199,1732670660585; forceNewPlan=false, retain=false 2024-11-27T01:24:36,701 INFO [7d6f26a4c136:43429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T01:24:36,701 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPENING, regionLocation=7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:36,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE because future has completed 2024-11-27T01:24:36,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585}] 2024-11-27T01:24:36,862 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,862 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 159a502e808ebaf9e21757c52ae64154, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:36,863 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,863 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:36,863 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,863 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,864 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,865 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf1 2024-11-27T01:24:36,866 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:36,872 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039 2024-11-27T01:24:36,872 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:36,873 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,873 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf2 2024-11-27T01:24:36,874 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:36,874 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:36,874 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,875 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,876 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,877 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,877 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,878 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:24:36,879 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,880 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 159a502e808ebaf9e21757c52ae64154; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71598303, jitterRate=0.06689785420894623}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:24:36,880 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:36,881 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 159a502e808ebaf9e21757c52ae64154: Running coprocessor pre-open hook at 1732670676863Writing region info on filesystem at 1732670676863Initializing all the Stores at 1732670676864 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670676864Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670676864Cleaning up temporary data from old regions at 1732670676877 (+13 ms)Running coprocessor post-open hooks at 1732670676880 (+3 ms)Region opened successfully at 1732670676880 2024-11-27T01:24:36,882 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., pid=9, masterSystemTime=1732670676857 2024-11-27T01:24:36,884 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,884 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:36,885 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPEN, openSeqNum=9, regionLocation=7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:36,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585 because future has completed 2024-11-27T01:24:36,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-27T01:24:36,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585 in 186 msec 2024-11-27T01:24:36,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE in 578 msec 2024-11-27T01:24:36,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:24:36,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:24:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:59150 deadline: 1732670736931, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=34199 startCode=1732670660585. As of locationSeqNum=5. 2024-11-27T01:24:36,963 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=34199 startCode=1732670660585. As of locationSeqNum=5. 2024-11-27T01:24:36,964 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=34199 startCode=1732670660585. As of locationSeqNum=5. 2024-11-27T01:24:36,964 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,34199,1732670660585, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=34199 startCode=1732670660585. As of locationSeqNum=5. 2024-11-27T01:24:37,072 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:24:37,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:24:37,083 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 159a502e808ebaf9e21757c52ae64154 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-27T01:24:37,100 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf1/9d0b9c50baca464c93e93fb854e0a790 is 29, key is r1/cf1:/1732670677075/DeleteFamily/seqid=0 2024-11-27T01:24:37,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741888_1066 (size=4906) 2024-11-27T01:24:37,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741888_1066 (size=4906) 2024-11-27T01:24:37,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741888_1066 (size=4906) 2024-11-27T01:24:37,109 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf1/9d0b9c50baca464c93e93fb854e0a790 2024-11-27T01:24:37,115 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9d0b9c50baca464c93e93fb854e0a790 2024-11-27T01:24:37,131 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf2/8c5ce1637ecd4c998b49b96b57c1c66a is 29, key is r1/cf2:/1732670677075/DeleteFamily/seqid=0 2024-11-27T01:24:37,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741889_1067 (size=4906) 2024-11-27T01:24:37,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741889_1067 (size=4906) 2024-11-27T01:24:37,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741889_1067 (size=4906) 2024-11-27T01:24:37,140 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf2/8c5ce1637ecd4c998b49b96b57c1c66a 2024-11-27T01:24:37,146 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c5ce1637ecd4c998b49b96b57c1c66a 2024-11-27T01:24:37,147 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf1/9d0b9c50baca464c93e93fb854e0a790 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/9d0b9c50baca464c93e93fb854e0a790 2024-11-27T01:24:37,154 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9d0b9c50baca464c93e93fb854e0a790 2024-11-27T01:24:37,155 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/9d0b9c50baca464c93e93fb854e0a790, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T01:24:37,156 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf2/8c5ce1637ecd4c998b49b96b57c1c66a as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/8c5ce1637ecd4c998b49b96b57c1c66a 2024-11-27T01:24:37,162 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c5ce1637ecd4c998b49b96b57c1c66a 2024-11-27T01:24:37,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/8c5ce1637ecd4c998b49b96b57c1c66a, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T01:24:37,163 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 159a502e808ebaf9e21757c52ae64154 in 80ms, sequenceid=12, compaction requested=false 2024-11-27T01:24:37,163 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 159a502e808ebaf9e21757c52ae64154: 2024-11-27T01:24:37,166 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:24:37,168 DEBUG [Time-limited test {}] regionserver.HStore(1541): 159a502e808ebaf9e21757c52ae64154/cf1 is initiating major compaction (all files) 2024-11-27T01:24:37,168 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:24:37,168 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:37,169 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 159a502e808ebaf9e21757c52ae64154/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,169 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039, hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/9d0b9c50baca464c93e93fb854e0a790] into tmpdir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp, totalSize=9.5 K 2024-11-27T01:24:37,170 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3cfdbb79da6042d09cf16ca0b6a24039, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732670676297 2024-11-27T01:24:37,171 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9d0b9c50baca464c93e93fb854e0a790, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T01:24:37,182 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 159a502e808ebaf9e21757c52ae64154#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:24:37,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741890_1068 (size=4626) 2024-11-27T01:24:37,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741890_1068 (size=4626) 2024-11-27T01:24:37,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741890_1068 (size=4626) 2024-11-27T01:24:37,197 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf1/4c7272e8e0e4473db7971896aefce317 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/4c7272e8e0e4473db7971896aefce317 2024-11-27T01:24:37,212 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 159a502e808ebaf9e21757c52ae64154/cf1 of 159a502e808ebaf9e21757c52ae64154 into 4c7272e8e0e4473db7971896aefce317(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:24:37,212 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 159a502e808ebaf9e21757c52ae64154: 2024-11-27T01:24:37,212 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-27T01:24:37,212 DEBUG [Time-limited test {}] regionserver.HStore(1541): 159a502e808ebaf9e21757c52ae64154/cf2 is initiating major compaction (all files) 2024-11-27T01:24:37,212 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:24:37,212 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:24:37,212 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 159a502e808ebaf9e21757c52ae64154/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,213 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/8c5ce1637ecd4c998b49b96b57c1c66a] into tmpdir=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp, totalSize=4.8 K 2024-11-27T01:24:37,213 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8c5ce1637ecd4c998b49b96b57c1c66a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T01:24:37,219 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 159a502e808ebaf9e21757c52ae64154#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:24:37,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741891_1069 (size=4592) 2024-11-27T01:24:37,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741891_1069 (size=4592) 2024-11-27T01:24:37,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741891_1069 (size=4592) 2024-11-27T01:24:37,233 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/.tmp/cf2/1ece285e8ab44fcb82831cc222c8f56b as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/1ece285e8ab44fcb82831cc222c8f56b 2024-11-27T01:24:37,240 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 159a502e808ebaf9e21757c52ae64154/cf2 of 159a502e808ebaf9e21757c52ae64154 into 1ece285e8ab44fcb82831cc222c8f56b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:24:37,240 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 159a502e808ebaf9e21757c52ae64154: 2024-11-27T01:24:37,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=159a502e808ebaf9e21757c52ae64154, source=7d6f26a4c136,34199,1732670660585, destination=7d6f26a4c136,42983,1732670660687, warming up region on 7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:37,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=159a502e808ebaf9e21757c52ae64154, source=7d6f26a4c136,34199,1732670660585, destination=7d6f26a4c136,42983,1732670660687, running balancer 2024-11-27T01:24:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE 2024-11-27T01:24:37,247 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE 2024-11-27T01:24:37,248 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=CLOSING, regionLocation=7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:37,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(7855): Warmup {ENCODED => 159a502e808ebaf9e21757c52ae64154, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:37,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:37,249 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,251 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf1 2024-11-27T01:24:37,251 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:37,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE because future has completed 2024-11-27T01:24:37,252 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:24:37,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585}] 2024-11-27T01:24:37,258 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039 2024-11-27T01:24:37,263 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/4c7272e8e0e4473db7971896aefce317 2024-11-27T01:24:37,268 INFO [StoreFileOpener-159a502e808ebaf9e21757c52ae64154-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9d0b9c50baca464c93e93fb854e0a790 2024-11-27T01:24:37,268 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/9d0b9c50baca464c93e93fb854e0a790 2024-11-27T01:24:37,268 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:37,269 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,269 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf2 2024-11-27T01:24:37,269 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:37,276 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/1ece285e8ab44fcb82831cc222c8f56b 2024-11-27T01:24:37,281 INFO [StoreFileOpener-159a502e808ebaf9e21757c52ae64154-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c5ce1637ecd4c998b49b96b57c1c66a 2024-11-27T01:24:37,281 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/8c5ce1637ecd4c998b49b96b57c1c66a 2024-11-27T01:24:37,281 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(1722): Closing 159a502e808ebaf9e21757c52ae64154, disabling compactions & flushes 2024-11-27T01:24:37,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. after waiting 0 ms 2024-11-27T01:24:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42983 {}] regionserver.HRegion(1676): Region close journal for 159a502e808ebaf9e21757c52ae64154: Waiting for close lock at 1732670677281Disabling compacts and flushes for region at 1732670677281Disabling writes for close at 1732670677282 (+1 ms)Writing region close event to WAL at 1732670677283 (+1 ms)Closed at 1732670677283 2024-11-27T01:24:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-27T01:24:37,406 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,406 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:24:37,406 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 159a502e808ebaf9e21757c52ae64154, disabling compactions & flushes 2024-11-27T01:24:37,406 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,406 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,406 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. after waiting 0 ms 2024-11-27T01:24:37,406 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,407 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039, hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/9d0b9c50baca464c93e93fb854e0a790] to archive 2024-11-27T01:24:37,410 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:24:37,414 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039 to hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/3cfdbb79da6042d09cf16ca0b6a24039 2024-11-27T01:24:37,416 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/9d0b9c50baca464c93e93fb854e0a790 to hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/9d0b9c50baca464c93e93fb854e0a790 2024-11-27T01:24:37,433 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/8c5ce1637ecd4c998b49b96b57c1c66a] to archive 2024-11-27T01:24:37,435 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:24:37,437 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/8c5ce1637ecd4c998b49b96b57c1c66a to hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/8c5ce1637ecd4c998b49b96b57c1c66a 2024-11-27T01:24:37,442 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-27T01:24:37,443 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,443 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 159a502e808ebaf9e21757c52ae64154: Waiting for close lock at 1732670677406Running coprocessor pre-close hooks at 1732670677406Disabling compacts and flushes for region at 1732670677406Disabling writes for close at 1732670677406Writing region close event to WAL at 1732670677438 (+32 ms)Running coprocessor post-close hooks at 1732670677442 (+4 ms)Closed at 1732670677443 (+1 ms) 2024-11-27T01:24:37,443 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 159a502e808ebaf9e21757c52ae64154 move to 7d6f26a4c136,42983,1732670660687 record at close sequenceid=12 2024-11-27T01:24:37,445 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,446 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=CLOSED 2024-11-27T01:24:37,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585 because future has completed 2024-11-27T01:24:37,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T01:24:37,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585 in 199 msec 2024-11-27T01:24:37,455 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE; state=CLOSED, location=7d6f26a4c136,42983,1732670660687; forceNewPlan=false, retain=false 2024-11-27T01:24:37,605 INFO [7d6f26a4c136:43429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T01:24:37,605 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPENING, regionLocation=7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:37,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE because future has completed 2024-11-27T01:24:37,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687}] 2024-11-27T01:24:37,766 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,766 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 159a502e808ebaf9e21757c52ae64154, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:37,766 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,766 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:37,766 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,766 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,768 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,769 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf1 2024-11-27T01:24:37,769 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:37,775 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/4c7272e8e0e4473db7971896aefce317 2024-11-27T01:24:37,775 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:37,775 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,776 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf2 2024-11-27T01:24:37,776 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:37,781 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/1ece285e8ab44fcb82831cc222c8f56b 2024-11-27T01:24:37,781 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:37,782 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,783 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,784 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,784 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,784 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,785 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:24:37,787 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,787 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 159a502e808ebaf9e21757c52ae64154; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66718506, jitterRate=-0.005816787481307983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:24:37,788 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,788 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 159a502e808ebaf9e21757c52ae64154: Running coprocessor pre-open hook at 1732670677766Writing region info on filesystem at 1732670677766Initializing all the Stores at 1732670677767 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670677767Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670677767Cleaning up temporary data from old regions at 1732670677785 (+18 ms)Running coprocessor post-open hooks at 1732670677788 (+3 ms)Region opened successfully at 1732670677788 2024-11-27T01:24:37,789 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., pid=12, masterSystemTime=1732670677761 2024-11-27T01:24:37,791 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,792 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,792 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPEN, openSeqNum=18, regionLocation=7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:37,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687 because future has completed 2024-11-27T01:24:37,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-27T01:24:37,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,42983,1732670660687 in 188 msec 2024-11-27T01:24:37,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, REOPEN/MOVE in 554 msec 2024-11-27T01:24:37,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:24:37,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:24:37,852 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 7d6f26a4c136,42983,1732670660687: testing ***** 2024-11-27T01:24:37,852 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T01:24:37,854 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-27T01:24:37,855 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-27T01:24:37,858 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-27T01:24:37,858 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-27T01:24:37,864 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 430190592 }, "NonHeapMemoryUsage": { "committed": 172032000, "init": 7667712, "max": -1, "used": 169501416 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 11, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 8, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 10, "ProcessCallTime_98th_percentile": 10, "ProcessCallTime_99th_percentile": 10, "ProcessCallTime_99.9th_percentile": 10, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 11, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 8, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 10, "TotalCallTime_98th_percentile": 10, "TotalCallTime_99th_percentile": 10, "TotalCallTime_99.9th_percentile": 10, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:61959", "tag.serverName": "7d6f26a4c136,42983,1732670660687", "tag.clusterId": "be445ce8-d3ee-4cc8-bc38-06bc3b180362", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732670660687, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-27T01:24:37,867 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43429 {}] master.MasterRpcServices(700): 7d6f26a4c136,42983,1732670660687 reported a fatal error: ***** ABORTING region server 7d6f26a4c136,42983,1732670660687: testing ***** 2024-11-27T01:24:37,869 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,42983,1732670660687' ***** 2024-11-27T01:24:37,869 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-27T01:24:37,869 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:24:37,870 INFO [RS:1;7d6f26a4c136:42983 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-27T01:24:37,870 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:24:37,870 INFO [RS:1;7d6f26a4c136:42983 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-27T01:24:37,870 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(3091): Received CLOSE for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,870 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(956): aborting server 7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:55640 deadline: 1732670737870, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=42983 startCode=1732670660687. As of locationSeqNum=12. 2024-11-27T01:24:37,870 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:24:37,871 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 159a502e808ebaf9e21757c52ae64154, disabling compactions & flushes 2024-11-27T01:24:37,871 INFO [RS:1;7d6f26a4c136:42983 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7d6f26a4c136:42983. 2024-11-27T01:24:37,871 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,871 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,871 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. after waiting 0 ms 2024-11-27T01:24:37,871 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,871 DEBUG [RS:1;7d6f26a4c136:42983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:24:37,871 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,34199,1732670660585, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,34199,1732670660585, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=42983 startCode=1732670660687. As of locationSeqNum=12. 2024-11-27T01:24:37,871 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,34199,1732670660585, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=42983 startCode=1732670660687. As of locationSeqNum=12. 2024-11-27T01:24:37,871 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,34199,1732670660585, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=42983 startCode=1732670660687. As of locationSeqNum=12. 2024-11-27T01:24:37,871 DEBUG [RS:1;7d6f26a4c136:42983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:37,872 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T01:24:37,872 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1325): Online Regions={159a502e808ebaf9e21757c52ae64154=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.} 2024-11-27T01:24:37,872 DEBUG [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1351): Waiting on 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:37,873 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,873 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 159a502e808ebaf9e21757c52ae64154: Waiting for close lock at 1732670677870Running coprocessor pre-close hooks at 1732670677870Disabling compacts and flushes for region at 1732670677870Disabling writes for close at 1732670677871 (+1 ms)Writing region close event to WAL at 1732670677872 (+1 ms)Running coprocessor post-close hooks at 1732670677872Closed at 1732670677873 (+1 ms) 2024-11-27T01:24:37,873 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:37,945 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:24:37,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7d6f26a4c136,42983,1732670660687 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:37,982 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T01:24:37,983 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7d6f26a4c136,42983,1732670660687 aborting 2024-11-27T01:24:37,983 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T01:24:37,983 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7d6f26a4c136,42983,1732670660687 aborting 2024-11-27T01:24:37,983 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=12 from cache 2024-11-27T01:24:38,072 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,42983,1732670660687; all regions closed. 2024-11-27T01:24:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741835_1011 (size=1405) 2024-11-27T01:24:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741835_1011 (size=1405) 2024-11-27T01:24:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741835_1011 (size=1405) 2024-11-27T01:24:38,077 DEBUG [RS:1;7d6f26a4c136:42983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:38,077 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:24:38,077 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:24:38,077 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:24:38,078 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:24:38,078 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:24:38,078 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:24:38,078 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:24:38,078 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:24:38,078 INFO [RS:1;7d6f26a4c136:42983 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42983 2024-11-27T01:24:38,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,42983,1732670660687 2024-11-27T01:24:38,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:24:38,085 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:24:38,088 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,42983,1732670660687] 2024-11-27T01:24:38,089 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,42983,1732670660687 already deleted, retry=false 2024-11-27T01:24:38,089 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 7d6f26a4c136,42983,1732670660687 on 7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:38,094 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 7d6f26a4c136,42983,1732670660687, splitWal=true, meta=false 2024-11-27T01:24:38,096 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 7d6f26a4c136,42983,1732670660687 (carryingMeta=false) 7d6f26a4c136,42983,1732670660687/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7c9344a2[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T01:24:38,096 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 7d6f26a4c136,42983,1732670660687, splitWal=true, meta=false 2024-11-27T01:24:38,098 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 7d6f26a4c136,42983,1732670660687 had 1 regions 2024-11-27T01:24:38,099 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 7d6f26a4c136,42983,1732670660687, splitWal=true, meta=false, isMeta: false 2024-11-27T01:24:38,101 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting 2024-11-27T01:24:38,102 INFO [PEWorker-2 {}] master.SplitWALManager(105): 7d6f26a4c136,42983,1732670660687 WAL count=1, meta=false 2024-11-27T01:24:38,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C42983%2C1732670660687.1732670662108}] 2024-11-27T01:24:38,110 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:38,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C42983%2C1732670660687.1732670662108, worker=7d6f26a4c136,39619,1732670660728}] 2024-11-27T01:24:38,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:38,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42983-0x1015706048a0002, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:38,188 INFO [RS:1;7d6f26a4c136:42983 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:24:38,189 INFO [RS:1;7d6f26a4c136:42983 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,42983,1732670660687; zookeeper connection closed. 2024-11-27T01:24:38,189 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5392e89b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5392e89b 2024-11-27T01:24:38,195 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18] 2024-11-27T01:24:38,197 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7d6f26a4c136:42983 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7d6f26a4c136/172.17.0.2:42983 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:38,197 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18, error=java.net.ConnectException: Call to address=7d6f26a4c136:42983 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7d6f26a4c136/172.17.0.2:42983 2024-11-27T01:24:38,197 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-27T01:24:38,197 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18 from cache 2024-11-27T01:24:38,198 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 7d6f26a4c136:42983 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7d6f26a4c136/172.17.0.2:42983 2024-11-27T01:24:38,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39619 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-27T01:24:38,293 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108, size=1.4 K (1405bytes) 2024-11-27T01:24:38,293 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 2024-11-27T01:24:38,293 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 after 0ms 2024-11-27T01:24:38,296 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:38,296 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 took 3ms 2024-11-27T01:24:38,303 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 159a502e808ebaf9e21757c52ae64154: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-27T01:24:38,303 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 so closing down 2024-11-27T01:24:38,303 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:38,303 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:38,303 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108, size=1.4 K, length=1405, corrupted=false, cancelled=false 2024-11-27T01:24:38,304 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108, journal: Splitting hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108, size=1.4 K (1405bytes) at 1732670678293Finishing writing output for hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 so closing down at 1732670678303 (+10 ms)3 split writer threads finished at 1732670678303Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108, size=1.4 K, length=1405, corrupted=false, cancelled=false at 1732670678303 2024-11-27T01:24:38,304 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 2024-11-27T01:24:38,305 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-27T01:24:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43429 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-27T01:24:38,311 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting/7d6f26a4c136%2C42983%2C1732670660687.1732670662108 to hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs 2024-11-27T01:24:38,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T01:24:38,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C42983%2C1732670660687.1732670662108, worker=7d6f26a4c136,39619,1732670660728 in 199 msec 2024-11-27T01:24:38,315 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:38,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T01:24:38,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C42983%2C1732670660687.1732670662108, worker=7d6f26a4c136,39619,1732670660728 in 212 msec 2024-11-27T01:24:38,320 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting dir is empty, no logs to split. 2024-11-27T01:24:38,320 INFO [PEWorker-2 {}] master.SplitWALManager(105): 7d6f26a4c136,42983,1732670660687 WAL count=0, meta=false 2024-11-27T01:24:38,320 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 7d6f26a4c136,42983,1732670660687 WAL splitting is done? wals=0, meta=false 2024-11-27T01:24:38,322 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 7d6f26a4c136,42983,1732670660687 failed, ignore...File hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/WALs/7d6f26a4c136,42983,1732670660687-splitting does not exist. 2024-11-27T01:24:38,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN}] 2024-11-27T01:24:38,325 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN 2024-11-27T01:24:38,326 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:24:38,477 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:24:38,477 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:24:38,477 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:24:38,477 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:24:38,477 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:24:38,477 INFO [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:24:38,477 INFO [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:24:38,477 DEBUG [7d6f26a4c136:43429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:24:38,478 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPENING, regionLocation=7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:38,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN because future has completed 2024-11-27T01:24:38,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585}] 2024-11-27T01:24:38,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18] 2024-11-27T01:24:38,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 7d6f26a4c136:42983 this server is in the failed servers list 2024-11-27T01:24:38,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=7d6f26a4c136:42983 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7d6f26a4c136:42983 2024-11-27T01:24:38,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7d6f26a4c136:42983 2024-11-27T01:24:38,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,42983,1732670660687, seqNum=18 from cache 2024-11-27T01:24:38,637 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:38,638 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 159a502e808ebaf9e21757c52ae64154, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:38,638 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,638 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:38,638 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,638 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,639 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,641 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf1 2024-11-27T01:24:38,641 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:38,647 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf1/4c7272e8e0e4473db7971896aefce317 2024-11-27T01:24:38,647 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:38,648 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,649 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 159a502e808ebaf9e21757c52ae64154 columnFamilyName cf2 2024-11-27T01:24:38,649 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:38,654 DEBUG [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/cf2/1ece285e8ab44fcb82831cc222c8f56b 2024-11-27T01:24:38,654 INFO [StoreOpener-159a502e808ebaf9e21757c52ae64154-1 {}] regionserver.HStore(327): Store=159a502e808ebaf9e21757c52ae64154/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:38,655 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,656 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,657 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,658 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:24:38,660 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,661 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 159a502e808ebaf9e21757c52ae64154; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75445702, jitterRate=0.1242285668849945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:24:38,661 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:38,662 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 159a502e808ebaf9e21757c52ae64154: Running coprocessor pre-open hook at 1732670678638Writing region info on filesystem at 1732670678638Initializing all the Stores at 1732670678639 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670678639Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670678639Cleaning up temporary data from old regions at 1732670678658 (+19 ms)Running coprocessor post-open hooks at 1732670678661 (+3 ms)Region opened successfully at 1732670678662 (+1 ms) 2024-11-27T01:24:38,663 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., pid=17, masterSystemTime=1732670678634 2024-11-27T01:24:38,665 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:38,666 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:38,666 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=159a502e808ebaf9e21757c52ae64154, regionState=OPEN, openSeqNum=18, regionLocation=7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:38,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585 because future has completed 2024-11-27T01:24:38,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-27T01:24:38,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 159a502e808ebaf9e21757c52ae64154, server=7d6f26a4c136,34199,1732670660585 in 189 msec 2024-11-27T01:24:38,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-27T01:24:38,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=159a502e808ebaf9e21757c52ae64154, ASSIGN in 349 msec 2024-11-27T01:24:38,676 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 7d6f26a4c136,42983,1732670660687 after splitting done 2024-11-27T01:24:38,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 7d6f26a4c136,42983,1732670660687, splitWal=true, meta=false in 586 msec 2024-11-27T01:24:39,024 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154., hostname=7d6f26a4c136,34199,1732670660585, seqNum=18] 2024-11-27T01:24:39,039 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 405) Potentially hanging thread: RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_756976327_22 at /127.0.0.1:49086 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_756976327_22 at /127.0.0.1:57262 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1234728491_22 at /127.0.0.1:37560 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1050 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=6234 (was 6274) 2024-11-27T01:24:39,041 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-27T01:24:39,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=6233 2024-11-27T01:24:39,053 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-27T01:24:39,069 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:39,070 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:39,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:39,073 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-52778364, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-52778364, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:39,086 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-52778364/hregion-52778364.1732670679074, exclude list is [], retry=0 2024-11-27T01:24:39,089 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:39,089 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:39,090 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:39,093 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-52778364/hregion-52778364.1732670679074 2024-11-27T01:24:39,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:39,093 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 51657dbf0320473bbf9b318b1526e15e, NAME => 'testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:39,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741893_1071 (size=67) 2024-11-27T01:24:39,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741893_1071 (size=67) 2024-11-27T01:24:39,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741893_1071 (size=67) 2024-11-27T01:24:39,104 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:39,106 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,107 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName a 2024-11-27T01:24:39,107 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,108 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,108 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,109 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName b 2024-11-27T01:24:39,109 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,110 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,110 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,111 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName c 2024-11-27T01:24:39,111 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,112 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,112 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,114 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,114 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,115 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:39,116 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,118 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:39,118 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 51657dbf0320473bbf9b318b1526e15e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66986346, jitterRate=-0.0018256604671478271}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:39,119 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 51657dbf0320473bbf9b318b1526e15e: Writing region info on filesystem at 1732670679104Initializing all the Stores at 1732670679105 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679105Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679105Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679105Cleaning up temporary data from old regions at 1732670679114 (+9 ms)Region opened successfully at 1732670679119 (+5 ms) 2024-11-27T01:24:39,119 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 51657dbf0320473bbf9b318b1526e15e, disabling compactions & flushes 2024-11-27T01:24:39,119 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,119 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,119 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. after waiting 0 ms 2024-11-27T01:24:39,119 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,120 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,120 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 51657dbf0320473bbf9b318b1526e15e: Waiting for close lock at 1732670679119Disabling compacts and flushes for region at 1732670679119Disabling writes for close at 1732670679119Writing region close event to WAL at 1732670679120 (+1 ms)Closed at 1732670679120 2024-11-27T01:24:39,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741892_1070 (size=95) 2024-11-27T01:24:39,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741892_1070 (size=95) 2024-11-27T01:24:39,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741892_1070 (size=95) 2024-11-27T01:24:39,125 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:39,125 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-52778364:(num 1732670679074) 2024-11-27T01:24:39,125 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:39,127 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:39,140 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127, exclude list is [], retry=0 2024-11-27T01:24:39,143 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:39,143 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:39,143 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:39,145 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127 2024-11-27T01:24:39,145 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:39,146 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 51657dbf0320473bbf9b318b1526e15e, NAME => 'testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:39,146 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:39,146 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,146 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,147 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,148 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName a 2024-11-27T01:24:39,148 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,149 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,149 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,150 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName b 2024-11-27T01:24:39,150 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,150 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,150 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,151 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName c 2024-11-27T01:24:39,151 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,151 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,151 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,153 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,154 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,154 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,155 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:39,156 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,157 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 51657dbf0320473bbf9b318b1526e15e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67025414, jitterRate=-0.001243501901626587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:39,157 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 51657dbf0320473bbf9b318b1526e15e: Writing region info on filesystem at 1732670679146Initializing all the Stores at 1732670679147 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679147Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679147Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679147Cleaning up temporary data from old regions at 1732670679154 (+7 ms)Region opened successfully at 1732670679157 (+3 ms) 2024-11-27T01:24:39,186 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 51657dbf0320473bbf9b318b1526e15e 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T01:24:39,203 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/a/a6555b6537be4958a677f07699bfb249 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732670679157/Put/seqid=0 2024-11-27T01:24:39,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741895_1073 (size=5958) 2024-11-27T01:24:39,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741895_1073 (size=5958) 2024-11-27T01:24:39,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741895_1073 (size=5958) 2024-11-27T01:24:39,213 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/a/a6555b6537be4958a677f07699bfb249 2024-11-27T01:24:39,236 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/b/1d12ec3bfb404af1aedaad51cbb6a27d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732670679165/Put/seqid=0 2024-11-27T01:24:39,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741896_1074 (size=5958) 2024-11-27T01:24:39,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741896_1074 (size=5958) 2024-11-27T01:24:39,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741896_1074 (size=5958) 2024-11-27T01:24:39,245 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/b/1d12ec3bfb404af1aedaad51cbb6a27d 2024-11-27T01:24:39,270 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/c/f8de632867004117bc7ec94c05abe94f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732670679173/Put/seqid=0 2024-11-27T01:24:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741897_1075 (size=5958) 2024-11-27T01:24:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741897_1075 (size=5958) 2024-11-27T01:24:39,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741897_1075 (size=5958) 2024-11-27T01:24:39,680 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/c/f8de632867004117bc7ec94c05abe94f 2024-11-27T01:24:39,686 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/a/a6555b6537be4958a677f07699bfb249 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/a/a6555b6537be4958a677f07699bfb249 2024-11-27T01:24:39,691 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/a/a6555b6537be4958a677f07699bfb249, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T01:24:39,692 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/b/1d12ec3bfb404af1aedaad51cbb6a27d as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/b/1d12ec3bfb404af1aedaad51cbb6a27d 2024-11-27T01:24:39,697 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/b/1d12ec3bfb404af1aedaad51cbb6a27d, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T01:24:39,697 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/c/f8de632867004117bc7ec94c05abe94f as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/c/f8de632867004117bc7ec94c05abe94f 2024-11-27T01:24:39,702 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/c/f8de632867004117bc7ec94c05abe94f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T01:24:39,703 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 51657dbf0320473bbf9b318b1526e15e in 518ms, sequenceid=33, compaction requested=false 2024-11-27T01:24:39,703 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 51657dbf0320473bbf9b318b1526e15e: 2024-11-27T01:24:39,703 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 51657dbf0320473bbf9b318b1526e15e, disabling compactions & flushes 2024-11-27T01:24:39,703 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,703 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,703 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. after waiting 0 ms 2024-11-27T01:24:39,703 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,704 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. 2024-11-27T01:24:39,704 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 51657dbf0320473bbf9b318b1526e15e: Waiting for close lock at 1732670679703Disabling compacts and flushes for region at 1732670679703Disabling writes for close at 1732670679703Writing region close event to WAL at 1732670679704 (+1 ms)Closed at 1732670679704 2024-11-27T01:24:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741894_1072 (size=3385) 2024-11-27T01:24:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741894_1072 (size=3385) 2024-11-27T01:24:39,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741894_1072 (size=3385) 2024-11-27T01:24:39,712 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/b/1d12ec3bfb404af1aedaad51cbb6a27d to hdfs://localhost:41069/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/b/1d12ec3bfb404af1aedaad51cbb6a27d 2024-11-27T01:24:39,729 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127, size=3.3 K (3385bytes) 2024-11-27T01:24:39,729 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127 2024-11-27T01:24:39,729 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127 after 0ms 2024-11-27T01:24:39,731 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:39,731 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127 took 3ms 2024-11-27T01:24:39,734 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127 so closing down 2024-11-27T01:24:39,734 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:39,735 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670679127.temp 2024-11-27T01:24:39,736 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000003-wal.1732670679127.temp 2024-11-27T01:24:39,736 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:39,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741898_1076 (size=2944) 2024-11-27T01:24:39,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741898_1076 (size=2944) 2024-11-27T01:24:39,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741898_1076 (size=2944) 2024-11-27T01:24:39,747 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000003-wal.1732670679127.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:39,748 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000003-wal.1732670679127.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000032 2024-11-27T01:24:39,748 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127, size=3.3 K, length=3385, corrupted=false, cancelled=false 2024-11-27T01:24:39,749 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127, journal: Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127, size=3.3 K (3385bytes) at 1732670679729Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127 so closing down at 1732670679734 (+5 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000003-wal.1732670679127.temp at 1732670679736 (+2 ms)3 split writer threads finished at 1732670679736Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000003-wal.1732670679127.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732670679747 (+11 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000003-wal.1732670679127.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000032 at 1732670679748 (+1 ms)Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127, size=3.3 K, length=3385, corrupted=false, cancelled=false at 1732670679749 (+1 ms) 2024-11-27T01:24:39,750 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679127 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670679127 2024-11-27T01:24:39,751 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000032 2024-11-27T01:24:39,751 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:39,753 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:39,774 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679753, exclude list is [], retry=0 2024-11-27T01:24:39,776 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:39,777 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:39,777 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:39,779 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679753 2024-11-27T01:24:39,779 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:39,779 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 51657dbf0320473bbf9b318b1526e15e, NAME => 'testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:39,780 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:39,780 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,780 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,781 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,782 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName a 2024-11-27T01:24:39,782 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,788 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/a/a6555b6537be4958a677f07699bfb249 2024-11-27T01:24:39,788 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,788 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,789 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName b 2024-11-27T01:24:39,790 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,790 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,790 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,791 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51657dbf0320473bbf9b318b1526e15e columnFamilyName c 2024-11-27T01:24:39,791 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,796 DEBUG [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/c/f8de632867004117bc7ec94c05abe94f 2024-11-27T01:24:39,796 INFO [StoreOpener-51657dbf0320473bbf9b318b1526e15e-1 {}] regionserver.HStore(327): Store=51657dbf0320473bbf9b318b1526e15e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,796 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,797 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,798 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,799 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000032 2024-11-27T01:24:39,801 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:39,803 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000032 2024-11-27T01:24:39,803 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 51657dbf0320473bbf9b318b1526e15e 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T01:24:39,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/b/2ab5e23b8e6944a7913eafe555b16539 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732670679165/Put/seqid=0 2024-11-27T01:24:39,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741900_1078 (size=5958) 2024-11-27T01:24:39,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741900_1078 (size=5958) 2024-11-27T01:24:39,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741900_1078 (size=5958) 2024-11-27T01:24:39,832 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/b/2ab5e23b8e6944a7913eafe555b16539 2024-11-27T01:24:39,841 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/.tmp/b/2ab5e23b8e6944a7913eafe555b16539 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/b/2ab5e23b8e6944a7913eafe555b16539 2024-11-27T01:24:39,846 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/b/2ab5e23b8e6944a7913eafe555b16539, entries=10, sequenceid=32, filesize=5.8 K 2024-11-27T01:24:39,847 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 51657dbf0320473bbf9b318b1526e15e in 44ms, sequenceid=32, compaction requested=false; wal=null 2024-11-27T01:24:39,847 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/0000000000000000032 2024-11-27T01:24:39,848 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,848 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,849 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:39,850 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 51657dbf0320473bbf9b318b1526e15e 2024-11-27T01:24:39,853 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/51657dbf0320473bbf9b318b1526e15e/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-27T01:24:39,854 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 51657dbf0320473bbf9b318b1526e15e; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68465925, jitterRate=0.020221784710884094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:39,854 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 51657dbf0320473bbf9b318b1526e15e: Writing region info on filesystem at 1732670679780Initializing all the Stores at 1732670679781 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679781Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679781Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679781Obtaining lock to block concurrent updates at 1732670679803 (+22 ms)Preparing flush snapshotting stores in 51657dbf0320473bbf9b318b1526e15e at 1732670679803Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732670679803Flushing stores of testReplayEditsWrittenViaHRegion,,1732670679069.51657dbf0320473bbf9b318b1526e15e. at 1732670679803Flushing 51657dbf0320473bbf9b318b1526e15e/b: creating writer at 1732670679803Flushing 51657dbf0320473bbf9b318b1526e15e/b: appending metadata at 1732670679823 (+20 ms)Flushing 51657dbf0320473bbf9b318b1526e15e/b: closing flushed file at 1732670679823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c63739e: reopening flushed file at 1732670679839 (+16 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 51657dbf0320473bbf9b318b1526e15e in 44ms, sequenceid=32, compaction requested=false; wal=null at 1732670679847 (+8 ms)Cleaning up temporary data from old regions at 1732670679848 (+1 ms)Region opened successfully at 1732670679854 (+6 ms) 2024-11-27T01:24:39,883 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=413 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:49086 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57262 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:49216 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:57392 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:37710 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:37560 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=6208 (was 6233) 2024-11-27T01:24:39,884 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-27T01:24:39,900 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=413, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=6207 2024-11-27T01:24:39,900 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-11-27T01:24:39,916 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:39,918 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:39,919 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:39,921 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-84321724, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-84321724, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:39,935 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-84321724/hregion-84321724.1732670679922, exclude list is [], retry=0 2024-11-27T01:24:39,938 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:39,938 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:39,940 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:39,942 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-84321724/hregion-84321724.1732670679922 2024-11-27T01:24:39,942 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:39,943 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 38f17031948cc1d87f1c88f3a400c237, NAME => 'testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:39,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741902_1080 (size=68) 2024-11-27T01:24:39,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741902_1080 (size=68) 2024-11-27T01:24:39,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741902_1080 (size=68) 2024-11-27T01:24:39,956 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:39,957 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,959 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName a 2024-11-27T01:24:39,959 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,960 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,960 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,961 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName b 2024-11-27T01:24:39,961 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,962 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,962 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,963 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName c 2024-11-27T01:24:39,963 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:39,963 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:39,964 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,964 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,965 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,966 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,966 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,966 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:39,968 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:39,970 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:39,970 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 38f17031948cc1d87f1c88f3a400c237; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62418183, jitterRate=-0.06989659368991852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:39,971 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 38f17031948cc1d87f1c88f3a400c237: Writing region info on filesystem at 1732670679956Initializing all the Stores at 1732670679957 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679957Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679957Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670679957Cleaning up temporary data from old regions at 1732670679966 (+9 ms)Region opened successfully at 1732670679971 (+5 ms) 2024-11-27T01:24:39,971 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 38f17031948cc1d87f1c88f3a400c237, disabling compactions & flushes 2024-11-27T01:24:39,971 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:39,972 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:39,972 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. after waiting 0 ms 2024-11-27T01:24:39,972 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:39,972 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:39,973 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 38f17031948cc1d87f1c88f3a400c237: Waiting for close lock at 1732670679971Disabling compacts and flushes for region at 1732670679971Disabling writes for close at 1732670679972 (+1 ms)Writing region close event to WAL at 1732670679972Closed at 1732670679972 2024-11-27T01:24:39,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741901_1079 (size=95) 2024-11-27T01:24:39,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741901_1079 (size=95) 2024-11-27T01:24:39,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741901_1079 (size=95) 2024-11-27T01:24:39,981 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:39,981 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-84321724:(num 1732670679922) 2024-11-27T01:24:39,981 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:39,983 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:39,997 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983, exclude list is [], retry=0 2024-11-27T01:24:39,999 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:39,999 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:40,000 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:40,001 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983 2024-11-27T01:24:40,002 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:40,067 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 38f17031948cc1d87f1c88f3a400c237, NAME => 'testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:40,069 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,070 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:40,070 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,070 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,072 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,073 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName a 2024-11-27T01:24:40,073 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:40,073 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:40,073 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,074 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName b 2024-11-27T01:24:40,074 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:40,075 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:40,075 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,076 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName c 2024-11-27T01:24:40,076 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:40,076 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:40,076 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,077 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,078 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,078 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,078 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,079 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:40,080 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,081 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 38f17031948cc1d87f1c88f3a400c237; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71200756, jitterRate=0.060973942279815674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:40,081 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:40,081 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 38f17031948cc1d87f1c88f3a400c237: Running coprocessor pre-open hook at 1732670680070Writing region info on filesystem at 1732670680070Initializing all the Stores at 1732670680071 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670680071Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670680071Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670680071Cleaning up temporary data from old regions at 1732670680078 (+7 ms)Running coprocessor post-open hooks at 1732670680081 (+3 ms)Region opened successfully at 1732670680081 2024-11-27T01:24:40,094 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 38f17031948cc1d87f1c88f3a400c237 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-27T01:24:40,095 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:40,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-27T01:24:40,298 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:40,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-27T01:24:40,299 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:40,300 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T01:24:40,300 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:40,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-27T01:24:40,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:40,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-27T01:24:40,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:41,096 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:41,558 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:24:42,097 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:43,098 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:44,098 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:45,099 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:46,099 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:47,100 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:48,101 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:48,975 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:24:49,101 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 38f17031948cc1d87f1c88f3a400c237/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:49,103 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 38f17031948cc1d87f1c88f3a400c237: 2024-11-27T01:24:49,103 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:49,117 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 38f17031948cc1d87f1c88f3a400c237: 2024-11-27T01:24:49,117 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-27T01:24:49,117 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 38f17031948cc1d87f1c88f3a400c237, disabling compactions & flushes 2024-11-27T01:24:49,117 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:49,117 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:49,117 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. after waiting 0 ms 2024-11-27T01:24:49,117 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:49,118 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:49,118 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. 2024-11-27T01:24:49,118 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 38f17031948cc1d87f1c88f3a400c237: Waiting for close lock at 1732670689117Running coprocessor pre-close hooks at 1732670689117Disabling compacts and flushes for region at 1732670689117Disabling writes for close at 1732670689117Writing region close event to WAL at 1732670689118 (+1 ms)Running coprocessor post-close hooks at 1732670689118Closed at 1732670689118 2024-11-27T01:24:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741903_1081 (size=2691) 2024-11-27T01:24:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741903_1081 (size=2691) 2024-11-27T01:24:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741903_1081 (size=2691) 2024-11-27T01:24:49,137 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983, size=2.6 K (2691bytes) 2024-11-27T01:24:49,137 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983 2024-11-27T01:24:49,138 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983 after 0ms 2024-11-27T01:24:49,140 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:49,140 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983 took 3ms 2024-11-27T01:24:49,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983 so closing down 2024-11-27T01:24:49,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:49,143 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732670679983.temp 2024-11-27T01:24:49,145 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000004-wal.1732670679983.temp 2024-11-27T01:24:49,145 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741904_1082 (size=2094) 2024-11-27T01:24:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741904_1082 (size=2094) 2024-11-27T01:24:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741904_1082 (size=2094) 2024-11-27T01:24:49,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000004-wal.1732670679983.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:49,159 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000004-wal.1732670679983.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000026 2024-11-27T01:24:49,159 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-11-27T01:24:49,160 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983, journal: Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983, size=2.6 K (2691bytes) at 1732670689137Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983 so closing down at 1732670689143 (+6 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000004-wal.1732670679983.temp at 1732670689145 (+2 ms)3 split writer threads finished at 1732670689145Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000004-wal.1732670679983.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732670689158 (+13 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000004-wal.1732670679983.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000026 at 1732670689159 (+1 ms)Processed 23 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1732670689159 2024-11-27T01:24:49,161 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670679983 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670679983 2024-11-27T01:24:49,162 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000026 2024-11-27T01:24:49,162 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:49,164 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:49,178 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670689164, exclude list is [], retry=0 2024-11-27T01:24:49,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:49,180 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:49,181 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:49,182 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670689164 2024-11-27T01:24:49,183 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:49,183 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 38f17031948cc1d87f1c88f3a400c237, NAME => 'testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:49,184 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,184 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:49,184 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,184 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,186 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,187 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName a 2024-11-27T01:24:49,187 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:49,188 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:49,188 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,189 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName b 2024-11-27T01:24:49,189 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:49,190 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:49,190 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,191 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38f17031948cc1d87f1c88f3a400c237 columnFamilyName c 2024-11-27T01:24:49,191 DEBUG [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:49,191 INFO [StoreOpener-38f17031948cc1d87f1c88f3a400c237-1 {}] regionserver.HStore(327): Store=38f17031948cc1d87f1c88f3a400c237/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:49,191 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,192 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,194 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000026 2024-11-27T01:24:49,197 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:49,198 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000026 2024-11-27T01:24:49,199 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 38f17031948cc1d87f1c88f3a400c237 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-27T01:24:49,221 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/a/4b4051b752ba4e1c8b43b28b7b8a3bd1 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732670689108/Put/seqid=0 2024-11-27T01:24:49,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741906_1084 (size=5523) 2024-11-27T01:24:49,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741906_1084 (size=5523) 2024-11-27T01:24:49,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741906_1084 (size=5523) 2024-11-27T01:24:49,229 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/a/4b4051b752ba4e1c8b43b28b7b8a3bd1 2024-11-27T01:24:49,251 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/b/a6d71fcf2be8439d94dfc6662b7ee7e2 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732670689103/Put/seqid=0 2024-11-27T01:24:49,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741907_1085 (size=5524) 2024-11-27T01:24:49,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741907_1085 (size=5524) 2024-11-27T01:24:49,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741907_1085 (size=5524) 2024-11-27T01:24:49,258 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/b/a6d71fcf2be8439d94dfc6662b7ee7e2 2024-11-27T01:24:49,277 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/c/653b4b88e7754d8ba66b4f4c5d6dcdbb is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732670689106/Put/seqid=0 2024-11-27T01:24:49,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741908_1086 (size=5457) 2024-11-27T01:24:49,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741908_1086 (size=5457) 2024-11-27T01:24:49,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741908_1086 (size=5457) 2024-11-27T01:24:49,284 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/c/653b4b88e7754d8ba66b4f4c5d6dcdbb 2024-11-27T01:24:49,289 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/a/4b4051b752ba4e1c8b43b28b7b8a3bd1 as hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/a/4b4051b752ba4e1c8b43b28b7b8a3bd1 2024-11-27T01:24:49,294 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/a/4b4051b752ba4e1c8b43b28b7b8a3bd1, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T01:24:49,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/b/a6d71fcf2be8439d94dfc6662b7ee7e2 as hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/b/a6d71fcf2be8439d94dfc6662b7ee7e2 2024-11-27T01:24:49,299 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/b/a6d71fcf2be8439d94dfc6662b7ee7e2, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T01:24:49,300 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/.tmp/c/653b4b88e7754d8ba66b4f4c5d6dcdbb as hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/c/653b4b88e7754d8ba66b4f4c5d6dcdbb 2024-11-27T01:24:49,304 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/c/653b4b88e7754d8ba66b4f4c5d6dcdbb, entries=6, sequenceid=26, filesize=5.3 K 2024-11-27T01:24:49,305 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 38f17031948cc1d87f1c88f3a400c237 in 106ms, sequenceid=26, compaction requested=false; wal=null 2024-11-27T01:24:49,305 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/0000000000000000026 2024-11-27T01:24:49,306 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,306 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,307 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:49,308 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,310 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsAfterAbortingFlush/38f17031948cc1d87f1c88f3a400c237/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-27T01:24:49,311 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 38f17031948cc1d87f1c88f3a400c237; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71118639, jitterRate=0.059750303626060486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:49,311 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38f17031948cc1d87f1c88f3a400c237 2024-11-27T01:24:49,312 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 38f17031948cc1d87f1c88f3a400c237: Running coprocessor pre-open hook at 1732670689184Writing region info on filesystem at 1732670689184Initializing all the Stores at 1732670689185 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670689185Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670689186 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670689186Obtaining lock to block concurrent updates at 1732670689199 (+13 ms)Preparing flush snapshotting stores in 38f17031948cc1d87f1c88f3a400c237 at 1732670689199Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732670689199Flushing stores of testReplayEditsAfterAbortingFlush,,1732670679917.38f17031948cc1d87f1c88f3a400c237. at 1732670689199Flushing 38f17031948cc1d87f1c88f3a400c237/a: creating writer at 1732670689199Flushing 38f17031948cc1d87f1c88f3a400c237/a: appending metadata at 1732670689221 (+22 ms)Flushing 38f17031948cc1d87f1c88f3a400c237/a: closing flushed file at 1732670689221Flushing 38f17031948cc1d87f1c88f3a400c237/b: creating writer at 1732670689235 (+14 ms)Flushing 38f17031948cc1d87f1c88f3a400c237/b: appending metadata at 1732670689251 (+16 ms)Flushing 38f17031948cc1d87f1c88f3a400c237/b: closing flushed file at 1732670689251Flushing 38f17031948cc1d87f1c88f3a400c237/c: creating writer at 1732670689263 (+12 ms)Flushing 38f17031948cc1d87f1c88f3a400c237/c: appending metadata at 1732670689277 (+14 ms)Flushing 38f17031948cc1d87f1c88f3a400c237/c: closing flushed file at 1732670689277Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@acbf9d4: reopening flushed file at 1732670689288 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37dfec4d: reopening flushed file at 1732670689294 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65808051: reopening flushed file at 1732670689299 (+5 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 38f17031948cc1d87f1c88f3a400c237 in 106ms, sequenceid=26, compaction requested=false; wal=null at 1732670689305 (+6 ms)Cleaning up temporary data from old regions at 1732670689306 (+1 ms)Running coprocessor post-open hooks at 1732670689311 (+5 ms)Region opened successfully at 1732670689311 2024-11-27T01:24:49,335 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=412 (was 413), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=205 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=6135 (was 6207) 2024-11-27T01:24:49,335 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-27T01:24:49,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=412, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=205, ProcessCount=11, AvailableMemoryMB=6134 2024-11-27T01:24:49,349 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-27T01:24:49,364 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:49,366 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:49,367 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:49,369 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78405619, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-78405619, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:49,382 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78405619/hregion-78405619.1732670689369, exclude list is [], retry=0 2024-11-27T01:24:49,385 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:49,385 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:49,385 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:49,387 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78405619/hregion-78405619.1732670689369 2024-11-27T01:24:49,387 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:49,387 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => de6af09c59dc9c5dde064beface88927, NAME => 'testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:49,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741910_1088 (size=61) 2024-11-27T01:24:49,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741910_1088 (size=61) 2024-11-27T01:24:49,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741910_1088 (size=61) 2024-11-27T01:24:49,396 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:49,397 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,398 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de6af09c59dc9c5dde064beface88927 columnFamilyName a 2024-11-27T01:24:49,398 DEBUG [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:49,399 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(327): Store=de6af09c59dc9c5dde064beface88927/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:49,399 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,400 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,400 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,400 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,400 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,401 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,403 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:49,403 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de6af09c59dc9c5dde064beface88927; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60630458, jitterRate=-0.09653577208518982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:49,404 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de6af09c59dc9c5dde064beface88927: Writing region info on filesystem at 1732670689396Initializing all the Stores at 1732670689397 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670689397Cleaning up temporary data from old regions at 1732670689400 (+3 ms)Region opened successfully at 1732670689404 (+4 ms) 2024-11-27T01:24:49,404 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de6af09c59dc9c5dde064beface88927, disabling compactions & flushes 2024-11-27T01:24:49,404 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,404 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,404 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. after waiting 0 ms 2024-11-27T01:24:49,404 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,405 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,405 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de6af09c59dc9c5dde064beface88927: Waiting for close lock at 1732670689404Disabling compacts and flushes for region at 1732670689404Disabling writes for close at 1732670689404Writing region close event to WAL at 1732670689405 (+1 ms)Closed at 1732670689405 2024-11-27T01:24:49,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741909_1087 (size=95) 2024-11-27T01:24:49,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741909_1087 (size=95) 2024-11-27T01:24:49,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741909_1087 (size=95) 2024-11-27T01:24:49,409 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:49,409 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78405619:(num 1732670689369) 2024-11-27T01:24:49,409 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:49,411 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:49,425 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411, exclude list is [], retry=0 2024-11-27T01:24:49,427 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:49,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:49,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:49,429 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 2024-11-27T01:24:49,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:49,430 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de6af09c59dc9c5dde064beface88927, NAME => 'testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:49,430 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:49,430 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,430 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,431 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,432 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de6af09c59dc9c5dde064beface88927 columnFamilyName a 2024-11-27T01:24:49,432 DEBUG [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:49,432 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(327): Store=de6af09c59dc9c5dde064beface88927/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:49,433 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,433 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,434 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,434 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,434 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,436 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,437 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de6af09c59dc9c5dde064beface88927; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74773261, jitterRate=0.1142084151506424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:49,437 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de6af09c59dc9c5dde064beface88927: Writing region info on filesystem at 1732670689430Initializing all the Stores at 1732670689431 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670689431Cleaning up temporary data from old regions at 1732670689435 (+4 ms)Region opened successfully at 1732670689437 (+2 ms) 2024-11-27T01:24:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de6af09c59dc9c5dde064beface88927, disabling compactions & flushes 2024-11-27T01:24:49,445 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. after waiting 0 ms 2024-11-27T01:24:49,445 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,445 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,445 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. 2024-11-27T01:24:49,446 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de6af09c59dc9c5dde064beface88927: Waiting for close lock at 1732670689445Disabling compacts and flushes for region at 1732670689445Disabling writes for close at 1732670689445Writing region close event to WAL at 1732670689445Closed at 1732670689445 2024-11-27T01:24:49,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741911_1089 (size=1050) 2024-11-27T01:24:49,449 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 not finished, retry = 0 2024-11-27T01:24:49,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741911_1089 (size=1050) 2024-11-27T01:24:49,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741911_1089 (size=1050) 2024-11-27T01:24:49,565 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411, size=1.0 K (1050bytes) 2024-11-27T01:24:49,565 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 2024-11-27T01:24:49,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 after 1ms 2024-11-27T01:24:49,568 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:49,569 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 took 4ms 2024-11-27T01:24:49,570 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 so closing down 2024-11-27T01:24:49,571 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:49,572 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670689411.temp 2024-11-27T01:24:49,573 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000003-wal.1732670689411.temp 2024-11-27T01:24:49,573 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:49,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741912_1090 (size=1050) 2024-11-27T01:24:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741912_1090 (size=1050) 2024-11-27T01:24:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741912_1090 (size=1050) 2024-11-27T01:24:49,582 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000003-wal.1732670689411.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:49,584 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000003-wal.1732670689411.temp to hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012 2024-11-27T01:24:49,584 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-27T01:24:49,584 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411, journal: Splitting hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411, size=1.0 K (1050bytes) at 1732670689565Finishing writing output for hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 so closing down at 1732670689570 (+5 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000003-wal.1732670689411.temp at 1732670689573 (+3 ms)3 split writer threads finished at 1732670689573Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000003-wal.1732670689411.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732670689582 (+9 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000003-wal.1732670689411.temp to hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012 at 1732670689584 (+2 ms)Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1732670689584 2024-11-27T01:24:49,585 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689411 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670689411 2024-11-27T01:24:49,586 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012 2024-11-27T01:24:49,589 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:49,928 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:49,931 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:49,952 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689932, exclude list is [], retry=0 2024-11-27T01:24:49,956 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:49,956 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:49,956 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:49,958 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689932 2024-11-27T01:24:49,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867)] 2024-11-27T01:24:49,958 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de6af09c59dc9c5dde064beface88927, NAME => 'testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:49,959 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:49,959 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,959 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,962 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,964 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de6af09c59dc9c5dde064beface88927 columnFamilyName a 2024-11-27T01:24:49,964 DEBUG [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:49,964 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(327): Store=de6af09c59dc9c5dde064beface88927/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:49,965 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,965 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:49,967 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012 2024-11-27T01:24:49,969 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:49,970 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012 2024-11-27T01:24:49,970 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing de6af09c59dc9c5dde064beface88927 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-27T01:24:49,996 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/.tmp/a/4a36dd5f1424491e847fc67338685837 is 79, key is testDatalossWhenInputError/a:x0/1732670689437/Put/seqid=0 2024-11-27T01:24:50,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741914_1092 (size=5808) 2024-11-27T01:24:50,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741914_1092 (size=5808) 2024-11-27T01:24:50,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741914_1092 (size=5808) 2024-11-27T01:24:50,004 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/.tmp/a/4a36dd5f1424491e847fc67338685837 2024-11-27T01:24:50,013 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/.tmp/a/4a36dd5f1424491e847fc67338685837 as hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/a/4a36dd5f1424491e847fc67338685837 2024-11-27T01:24:50,022 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/a/4a36dd5f1424491e847fc67338685837, entries=10, sequenceid=12, filesize=5.7 K 2024-11-27T01:24:50,022 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for de6af09c59dc9c5dde064beface88927 in 52ms, sequenceid=12, compaction requested=false; wal=null 2024-11-27T01:24:50,023 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/0000000000000000012 2024-11-27T01:24:50,024 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,024 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,027 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,029 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-27T01:24:50,030 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de6af09c59dc9c5dde064beface88927; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63946756, jitterRate=-0.047119081020355225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:50,030 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de6af09c59dc9c5dde064beface88927: Writing region info on filesystem at 1732670689959Initializing all the Stores at 1732670689962 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670689962Obtaining lock to block concurrent updates at 1732670689970 (+8 ms)Preparing flush snapshotting stores in de6af09c59dc9c5dde064beface88927 at 1732670689970Finished memstore snapshotting testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732670689970Flushing stores of testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927. at 1732670689970Flushing de6af09c59dc9c5dde064beface88927/a: creating writer at 1732670689971 (+1 ms)Flushing de6af09c59dc9c5dde064beface88927/a: appending metadata at 1732670689995 (+24 ms)Flushing de6af09c59dc9c5dde064beface88927/a: closing flushed file at 1732670689995Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73cea091: reopening flushed file at 1732670690011 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for de6af09c59dc9c5dde064beface88927 in 52ms, sequenceid=12, compaction requested=false; wal=null at 1732670690022 (+11 ms)Cleaning up temporary data from old regions at 1732670690024 (+2 ms)Region opened successfully at 1732670690030 (+6 ms) 2024-11-27T01:24:50,033 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de6af09c59dc9c5dde064beface88927, NAME => 'testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:50,034 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670689365.de6af09c59dc9c5dde064beface88927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:50,034 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,034 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,035 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,036 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de6af09c59dc9c5dde064beface88927 columnFamilyName a 2024-11-27T01:24:50,036 DEBUG [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:50,042 DEBUG [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/a/4a36dd5f1424491e847fc67338685837 2024-11-27T01:24:50,042 INFO [StoreOpener-de6af09c59dc9c5dde064beface88927-1 {}] regionserver.HStore(327): Store=de6af09c59dc9c5dde064beface88927/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:50,042 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,043 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,045 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,045 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,047 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de6af09c59dc9c5dde064beface88927 2024-11-27T01:24:50,049 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testDatalossWhenInputError/de6af09c59dc9c5dde064beface88927/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-27T01:24:50,050 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de6af09c59dc9c5dde064beface88927; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60986550, jitterRate=-0.09122958779335022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:24:50,050 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de6af09c59dc9c5dde064beface88927: Writing region info on filesystem at 1732670690034Initializing all the Stores at 1732670690035 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670690035Cleaning up temporary data from old regions at 1732670690045 (+10 ms)Region opened successfully at 1732670690050 (+5 ms) 2024-11-27T01:24:50,067 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=422 (was 412) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:50318 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:43246 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:56894 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:50392 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:56810 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-272052841-172.17.0.2-1732670656425:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:43308 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=205 (was 205), ProcessCount=11 (was 11), AvailableMemoryMB=6119 (was 6134) 2024-11-27T01:24:50,067 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-27T01:24:50,079 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=422, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=205, ProcessCount=11, AvailableMemoryMB=6118 2024-11-27T01:24:50,079 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-27T01:24:50,094 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:50,096 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:50,097 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:50,099 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54285608, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-54285608, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:50,112 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54285608/hregion-54285608.1732670690099, exclude list is [], retry=0 2024-11-27T01:24:50,115 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:50,115 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:50,115 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:50,117 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54285608/hregion-54285608.1732670690099 2024-11-27T01:24:50,117 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:50,117 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c98e24e9e93b660966d65c855fe2e0c4, NAME => 'testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:50,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741916_1094 (size=63) 2024-11-27T01:24:50,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741916_1094 (size=63) 2024-11-27T01:24:50,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741916_1094 (size=63) 2024-11-27T01:24:50,129 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:50,130 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,132 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName a 2024-11-27T01:24:50,132 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:50,132 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:50,133 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,134 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName b 2024-11-27T01:24:50,134 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:50,134 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:50,135 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,136 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName c 2024-11-27T01:24:50,136 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:50,137 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:50,137 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,139 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,140 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,141 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,141 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,141 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:50,143 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,145 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:50,145 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c98e24e9e93b660966d65c855fe2e0c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69456248, jitterRate=0.03497874736785889}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:50,146 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c98e24e9e93b660966d65c855fe2e0c4: Writing region info on filesystem at 1732670690129Initializing all the Stores at 1732670690130 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670690130Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670690130Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670690130Cleaning up temporary data from old regions at 1732670690141 (+11 ms)Region opened successfully at 1732670690146 (+5 ms) 2024-11-27T01:24:50,146 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c98e24e9e93b660966d65c855fe2e0c4, disabling compactions & flushes 2024-11-27T01:24:50,146 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:50,146 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:50,146 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. after waiting 0 ms 2024-11-27T01:24:50,146 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:50,146 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:50,146 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c98e24e9e93b660966d65c855fe2e0c4: Waiting for close lock at 1732670690146Disabling compacts and flushes for region at 1732670690146Disabling writes for close at 1732670690146Writing region close event to WAL at 1732670690146Closed at 1732670690146 2024-11-27T01:24:50,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741915_1093 (size=95) 2024-11-27T01:24:50,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741915_1093 (size=95) 2024-11-27T01:24:50,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741915_1093 (size=95) 2024-11-27T01:24:50,154 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:50,155 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54285608:(num 1732670690099) 2024-11-27T01:24:50,155 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:50,156 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:50,170 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157, exclude list is [], retry=0 2024-11-27T01:24:50,173 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:50,173 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:50,174 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:50,176 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 2024-11-27T01:24:50,176 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:50,177 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c98e24e9e93b660966d65c855fe2e0c4, NAME => 'testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:50,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:50,177 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,177 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,178 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,179 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName a 2024-11-27T01:24:50,179 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:50,180 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:50,180 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,181 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName b 2024-11-27T01:24:50,181 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:50,181 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:50,181 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,182 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName c 2024-11-27T01:24:50,182 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:50,182 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:50,182 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,183 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,184 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,185 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,185 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,185 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:50,187 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:50,188 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c98e24e9e93b660966d65c855fe2e0c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70520539, jitterRate=0.0508379191160202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:50,188 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c98e24e9e93b660966d65c855fe2e0c4: Writing region info on filesystem at 1732670690177Initializing all the Stores at 1732670690178 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670690178Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670690178Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670690178Cleaning up temporary data from old regions at 1732670690185 (+7 ms)Region opened successfully at 1732670690188 (+3 ms) 2024-11-27T01:24:50,192 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732670690192/Put/seqid=0 2024-11-27T01:24:50,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741918_1096 (size=4875) 2024-11-27T01:24:50,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741918_1096 (size=4875) 2024-11-27T01:24:50,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741918_1096 (size=4875) 2024-11-27T01:24:50,207 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732670690206/Put/seqid=0 2024-11-27T01:24:50,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741919_1097 (size=4875) 2024-11-27T01:24:50,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741919_1097 (size=4875) 2024-11-27T01:24:50,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741919_1097 (size=4875) 2024-11-27T01:24:50,220 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732670690219/Put/seqid=0 2024-11-27T01:24:50,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741920_1098 (size=4875) 2024-11-27T01:24:50,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741920_1098 (size=4875) 2024-11-27T01:24:50,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741920_1098 (size=4875) 2024-11-27T01:24:50,227 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c98e24e9e93b660966d65c855fe2e0c4/a 2024-11-27T01:24:50,230 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-27T01:24:50,230 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:24:50,231 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c98e24e9e93b660966d65c855fe2e0c4/a 2024-11-27T01:24:50,234 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-27T01:24:50,234 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:24:50,234 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c98e24e9e93b660966d65c855fe2e0c4/a 2024-11-27T01:24:50,238 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-27T01:24:50,238 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:24:50,238 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c98e24e9e93b660966d65c855fe2e0c4 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-27T01:24:50,255 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/.tmp/a/393e9ed9163b48c29e3e9fe33314ee59 is 55, key is testCompactedBulkLoadedFiles/a:a/1732670690189/Put/seqid=0 2024-11-27T01:24:50,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741921_1099 (size=5107) 2024-11-27T01:24:50,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741921_1099 (size=5107) 2024-11-27T01:24:50,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741921_1099 (size=5107) 2024-11-27T01:24:50,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/.tmp/a/393e9ed9163b48c29e3e9fe33314ee59 2024-11-27T01:24:50,271 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/.tmp/a/393e9ed9163b48c29e3e9fe33314ee59 as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59 2024-11-27T01:24:50,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59, entries=1, sequenceid=4, filesize=5.0 K 2024-11-27T01:24:50,278 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c98e24e9e93b660966d65c855fe2e0c4 in 40ms, sequenceid=4, compaction requested=false 2024-11-27T01:24:50,278 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c98e24e9e93b660966d65c855fe2e0c4: 2024-11-27T01:24:50,280 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_ 2024-11-27T01:24:50,281 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_ 2024-11-27T01:24:50,282 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_ 2024-11-27T01:24:50,282 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile0 into c98e24e9e93b660966d65c855fe2e0c4/a as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_ - updating store file list. 2024-11-27T01:24:50,286 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:50,286 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_ into c98e24e9e93b660966d65c855fe2e0c4/a 2024-11-27T01:24:50,286 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile0 into c98e24e9e93b660966d65c855fe2e0c4/a (new location: hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_) 2024-11-27T01:24:50,287 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile1 into c98e24e9e93b660966d65c855fe2e0c4/a as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_ - updating store file list. 2024-11-27T01:24:50,291 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c69470f86dbe43d7bf98300568cee8d5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:50,291 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_ into c98e24e9e93b660966d65c855fe2e0c4/a 2024-11-27T01:24:50,291 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile1 into c98e24e9e93b660966d65c855fe2e0c4/a (new location: hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_) 2024-11-27T01:24:50,292 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile2 into c98e24e9e93b660966d65c855fe2e0c4/a as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_ - updating store file list. 2024-11-27T01:24:50,296 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 498560210f5d45a596a89bff2d781f30_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:50,296 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_ into c98e24e9e93b660966d65c855fe2e0c4/a 2024-11-27T01:24:50,296 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41069/hbase/testCompactedBulkLoadedFiles/hfile2 into c98e24e9e93b660966d65c855fe2e0c4/a (new location: hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_) 2024-11-27T01:24:50,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-27T01:24:50,298 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:50,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-27T01:24:50,299 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-27T01:24:50,303 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:24:50,304 DEBUG [Time-limited test {}] regionserver.HStore(1541): c98e24e9e93b660966d65c855fe2e0c4/a is initiating major compaction (all files) 2024-11-27T01:24:50,304 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c98e24e9e93b660966d65c855fe2e0c4/a in testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:50,304 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_] into tmpdir=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/.tmp, totalSize=19.3 K 2024-11-27T01:24:50,304 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 393e9ed9163b48c29e3e9fe33314ee59, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732670690189 2024-11-27T01:24:50,305 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T01:24:50,305 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c69470f86dbe43d7bf98300568cee8d5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T01:24:50,306 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 498560210f5d45a596a89bff2d781f30_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T01:24:50,318 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/.tmp/a/3a8026d7c57c4997a4fb97f920d54302 is 55, key is testCompactedBulkLoadedFiles/a:a/1732670690189/Put/seqid=0 2024-11-27T01:24:50,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741922_1100 (size=6154) 2024-11-27T01:24:50,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741922_1100 (size=6154) 2024-11-27T01:24:50,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741922_1100 (size=6154) 2024-11-27T01:24:50,333 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/.tmp/a/3a8026d7c57c4997a4fb97f920d54302 as hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/3a8026d7c57c4997a4fb97f920d54302 2024-11-27T01:24:50,340 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c98e24e9e93b660966d65c855fe2e0c4/a of c98e24e9e93b660966d65c855fe2e0c4 into 3a8026d7c57c4997a4fb97f920d54302(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:24:50,340 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c98e24e9e93b660966d65c855fe2e0c4: 2024-11-27T01:24:50,340 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T01:24:50,340 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T01:24:50,374 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157, size=0 (0bytes) 2024-11-27T01:24:50,374 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 might be still open, length is 0 2024-11-27T01:24:50,374 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 2024-11-27T01:24:50,375 WARN [IPC Server handler 4 on default port 41069 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-27T01:24:50,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 after 1ms 2024-11-27T01:24:51,755 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:56932 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56932 dst: /127.0.0.1:37205 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37205 remote=/127.0.0.1:56932]. Total timeout mills is 60000, 58585 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:51,755 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:43348 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36269:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43348 dst: /127.0.0.1:36269 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:51,755 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:50434 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45195:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50434 dst: /127.0.0.1:45195 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:51,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741917_1101 (size=1168) 2024-11-27T01:24:51,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741917_1101 (size=1168) 2024-11-27T01:24:51,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741917_1101 (size=1168) 2024-11-27T01:24:54,376 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 after 4002ms 2024-11-27T01:24:54,379 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:54,379 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 took 4005ms 2024-11-27T01:24:54,381 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157; continuing. 2024-11-27T01:24:54,381 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 so closing down 2024-11-27T01:24:54,381 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:54,382 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670690157.temp 2024-11-27T01:24:54,383 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000003-wal.1732670690157.temp 2024-11-27T01:24:54,384 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:54,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741923_1102 (size=544) 2024-11-27T01:24:54,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741923_1102 (size=544) 2024-11-27T01:24:54,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741923_1102 (size=544) 2024-11-27T01:24:54,392 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000003-wal.1732670690157.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:54,393 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000003-wal.1732670690157.temp to hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000008 2024-11-27T01:24:54,393 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:24:54,393 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157, journal: Splitting hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157, size=0 (0bytes) at 1732670690374Finishing writing output for hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 so closing down at 1732670694381 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000003-wal.1732670690157.temp at 1732670694383 (+2 ms)3 split writer threads finished at 1732670694384 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000003-wal.1732670690157.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732670694392 (+8 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000003-wal.1732670690157.temp to hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000008 at 1732670694393 (+1 ms)Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157, size=0, length=0, corrupted=false, cancelled=false at 1732670694393 2024-11-27T01:24:54,395 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670690157 2024-11-27T01:24:54,396 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000008 2024-11-27T01:24:54,396 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:54,398 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:54,411 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670694398, exclude list is [], retry=0 2024-11-27T01:24:54,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:54,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:54,415 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:54,416 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670694398 2024-11-27T01:24:54,416 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867)] 2024-11-27T01:24:54,417 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c98e24e9e93b660966d65c855fe2e0c4, NAME => 'testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:54,417 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:54,417 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,417 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,419 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,419 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName a 2024-11-27T01:24:54,420 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,429 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59 2024-11-27T01:24:54,433 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/3a8026d7c57c4997a4fb97f920d54302 2024-11-27T01:24:54,436 DEBUG [StoreFileOpener-c98e24e9e93b660966d65c855fe2e0c4-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 498560210f5d45a596a89bff2d781f30_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:54,436 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_ 2024-11-27T01:24:54,439 DEBUG [StoreFileOpener-c98e24e9e93b660966d65c855fe2e0c4-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:54,440 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_ 2024-11-27T01:24:54,443 DEBUG [StoreFileOpener-c98e24e9e93b660966d65c855fe2e0c4-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c69470f86dbe43d7bf98300568cee8d5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:24:54,443 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_ 2024-11-27T01:24:54,443 WARN [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-11-27T01:24:54,443 WARN [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-11-27T01:24:54,443 WARN [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-11-27T01:24:54,443 WARN [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-11-27T01:24:54,443 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_] to archive 2024-11-27T01:24:54,444 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:24:54,446 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59 to hdfs://localhost:41069/hbase/archive/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/393e9ed9163b48c29e3e9fe33314ee59 2024-11-27T01:24:54,447 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_ to hdfs://localhost:41069/hbase/archive/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/498560210f5d45a596a89bff2d781f30_SeqId_4_ 2024-11-27T01:24:54,448 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_ to hdfs://localhost:41069/hbase/archive/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_ 2024-11-27T01:24:54,449 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_ to hdfs://localhost:41069/hbase/archive/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/a/c69470f86dbe43d7bf98300568cee8d5_SeqId_4_ 2024-11-27T01:24:54,449 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,449 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,450 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName b 2024-11-27T01:24:54,450 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,450 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,450 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,451 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c98e24e9e93b660966d65c855fe2e0c4 columnFamilyName c 2024-11-27T01:24:54,451 DEBUG [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,452 INFO [StoreOpener-c98e24e9e93b660966d65c855fe2e0c4-1 {}] regionserver.HStore(327): Store=c98e24e9e93b660966d65c855fe2e0c4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,452 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,452 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,454 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,454 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000008 2024-11-27T01:24:54,455 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:54,457 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c98e24e9e93b660966d65c855fe2e0c4 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c98e24e9e93b660966d65c855fe2e0c4" family_name: "a" compaction_input: "393e9ed9163b48c29e3e9fe33314ee59" compaction_input: "c6753b1d76364c1bb471d5831ac7b13f_SeqId_4_" compaction_input: "c69470f86dbe43d7bf98300568cee8d5_SeqId_4_" compaction_input: "498560210f5d45a596a89bff2d781f30_SeqId_4_" compaction_output: "3a8026d7c57c4997a4fb97f920d54302" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-27T01:24:54,457 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-27T01:24:54,457 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000008 2024-11-27T01:24:54,458 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/0000000000000000008 2024-11-27T01:24:54,459 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,459 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,459 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:54,461 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c98e24e9e93b660966d65c855fe2e0c4 2024-11-27T01:24:54,463 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testCompactedBulkLoadedFiles/c98e24e9e93b660966d65c855fe2e0c4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T01:24:54,464 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c98e24e9e93b660966d65c855fe2e0c4; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68159398, jitterRate=0.015654176473617554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:54,464 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c98e24e9e93b660966d65c855fe2e0c4: Writing region info on filesystem at 1732670694417Initializing all the Stores at 1732670694418 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694418Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694418Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694418Cleaning up temporary data from old regions at 1732670694459 (+41 ms)Region opened successfully at 1732670694464 (+5 ms) 2024-11-27T01:24:54,466 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c98e24e9e93b660966d65c855fe2e0c4, disabling compactions & flushes 2024-11-27T01:24:54,466 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:54,466 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:54,466 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. after waiting 0 ms 2024-11-27T01:24:54,466 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:54,467 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732670690095.c98e24e9e93b660966d65c855fe2e0c4. 2024-11-27T01:24:54,467 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c98e24e9e93b660966d65c855fe2e0c4: Waiting for close lock at 1732670694466Disabling compacts and flushes for region at 1732670694466Disabling writes for close at 1732670694466Writing region close event to WAL at 1732670694467 (+1 ms)Closed at 1732670694467 2024-11-27T01:24:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741924_1103 (size=95) 2024-11-27T01:24:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741924_1103 (size=95) 2024-11-27T01:24:54,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741924_1103 (size=95) 2024-11-27T01:24:54,471 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:54,471 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670694398) 2024-11-27T01:24:54,485 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=436 (was 422) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1774832404) connection to localhost/127.0.0.1:41069 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1440677955_22 at /127.0.0.1:52664 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1440677955_22 at /127.0.0.1:50082 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1440677955_22 at /127.0.0.1:56564 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:41069 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 205) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6024 (was 6118) 2024-11-27T01:24:54,486 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-27T01:24:54,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=436, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=6023 2024-11-27T01:24:54,498 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-27T01:24:54,513 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:54,515 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:24:54,516 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:24:54,518 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21218764, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/hregion-21218764, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:54,532 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21218764/hregion-21218764.1732670694518, exclude list is [], retry=0 2024-11-27T01:24:54,535 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:54,535 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:54,536 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:54,538 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21218764/hregion-21218764.1732670694518 2024-11-27T01:24:54,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:54,538 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bfa43c595f131a8070100a905a27a0b8, NAME => 'testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41069/hbase 2024-11-27T01:24:54,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741926_1105 (size=67) 2024-11-27T01:24:54,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741926_1105 (size=67) 2024-11-27T01:24:54,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741926_1105 (size=67) 2024-11-27T01:24:54,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:54,550 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,552 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName a 2024-11-27T01:24:54,552 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,552 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,552 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,554 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName b 2024-11-27T01:24:54,554 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,554 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,554 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,555 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName c 2024-11-27T01:24:54,555 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,556 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,556 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,557 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,557 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,558 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,558 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,558 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:54,559 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,562 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:24:54,562 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfa43c595f131a8070100a905a27a0b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66941199, jitterRate=-0.0024984031915664673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:54,562 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfa43c595f131a8070100a905a27a0b8: Writing region info on filesystem at 1732670694549Initializing all the Stores at 1732670694550 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694550Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694550Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694550Cleaning up temporary data from old regions at 1732670694558 (+8 ms)Region opened successfully at 1732670694562 (+4 ms) 2024-11-27T01:24:54,562 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bfa43c595f131a8070100a905a27a0b8, disabling compactions & flushes 2024-11-27T01:24:54,562 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,562 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,562 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. after waiting 0 ms 2024-11-27T01:24:54,562 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,563 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,563 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bfa43c595f131a8070100a905a27a0b8: Waiting for close lock at 1732670694562Disabling compacts and flushes for region at 1732670694562Disabling writes for close at 1732670694562Writing region close event to WAL at 1732670694563 (+1 ms)Closed at 1732670694563 2024-11-27T01:24:54,565 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-21218764/hregion-21218764.1732670694518 not finished, retry = 0 2024-11-27T01:24:54,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741925_1104 (size=95) 2024-11-27T01:24:54,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741925_1104 (size=95) 2024-11-27T01:24:54,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741925_1104 (size=95) 2024-11-27T01:24:54,639 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:24:54,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:54,668 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21218764:(num 1732670694518) 2024-11-27T01:24:54,668 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:54,670 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:54,684 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671, exclude list is [], retry=0 2024-11-27T01:24:54,686 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:54,687 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:54,687 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:54,688 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671 2024-11-27T01:24:54,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585)] 2024-11-27T01:24:54,689 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bfa43c595f131a8070100a905a27a0b8, NAME => 'testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:54,689 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:54,689 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,689 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,690 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,691 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName a 2024-11-27T01:24:54,691 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,692 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,692 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,693 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName b 2024-11-27T01:24:54,693 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,694 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,694 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,695 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName c 2024-11-27T01:24:54,695 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,695 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,695 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,697 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,698 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,698 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,699 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:54,700 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,701 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfa43c595f131a8070100a905a27a0b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59374021, jitterRate=-0.11525814235210419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:54,701 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfa43c595f131a8070100a905a27a0b8: Writing region info on filesystem at 1732670694689Initializing all the Stores at 1732670694690 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694690Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694690Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694690Cleaning up temporary data from old regions at 1732670694698 (+8 ms)Region opened successfully at 1732670694701 (+3 ms) 2024-11-27T01:24:54,711 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfa43c595f131a8070100a905a27a0b8 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T01:24:54,735 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/a/a1724476fedb4978b9129e5a78981e82 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732670694701/Put/seqid=0 2024-11-27T01:24:54,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741928_1107 (size=5958) 2024-11-27T01:24:54,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741928_1107 (size=5958) 2024-11-27T01:24:54,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741928_1107 (size=5958) 2024-11-27T01:24:54,743 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/a/a1724476fedb4978b9129e5a78981e82 2024-11-27T01:24:54,748 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/a/a1724476fedb4978b9129e5a78981e82 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/a/a1724476fedb4978b9129e5a78981e82 2024-11-27T01:24:54,753 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/a/a1724476fedb4978b9129e5a78981e82, entries=10, sequenceid=13, filesize=5.8 K 2024-11-27T01:24:54,754 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for bfa43c595f131a8070100a905a27a0b8 in 43ms, sequenceid=13, compaction requested=false 2024-11-27T01:24:54,754 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bfa43c595f131a8070100a905a27a0b8: 2024-11-27T01:24:54,773 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bfa43c595f131a8070100a905a27a0b8, disabling compactions & flushes 2024-11-27T01:24:54,773 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,773 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,773 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. after waiting 0 ms 2024-11-27T01:24:54,773 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,774 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,774 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:54,774 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bfa43c595f131a8070100a905a27a0b8: Waiting for close lock at 1732670694773Disabling compacts and flushes for region at 1732670694773Disabling writes for close at 1732670694773Writing region close event to WAL at 1732670694774 (+1 ms)Closed at 1732670694774 2024-11-27T01:24:54,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741927_1106 (size=3346) 2024-11-27T01:24:54,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741927_1106 (size=3346) 2024-11-27T01:24:54,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741927_1106 (size=3346) 2024-11-27T01:24:54,799 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671, size=3.3 K (3346bytes) 2024-11-27T01:24:54,799 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671 2024-11-27T01:24:54,799 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671 after 0ms 2024-11-27T01:24:54,801 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:54,802 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671 took 3ms 2024-11-27T01:24:54,804 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671 so closing down 2024-11-27T01:24:54,804 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:54,805 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670694671.temp 2024-11-27T01:24:54,806 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000003-wal.1732670694671.temp 2024-11-27T01:24:54,806 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:54,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741929_1108 (size=2944) 2024-11-27T01:24:54,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741929_1108 (size=2944) 2024-11-27T01:24:54,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741929_1108 (size=2944) 2024-11-27T01:24:54,813 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000003-wal.1732670694671.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:54,814 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000003-wal.1732670694671.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000035 2024-11-27T01:24:54,815 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671, size=3.3 K, length=3346, corrupted=false, cancelled=false 2024-11-27T01:24:54,815 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671, journal: Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671, size=3.3 K (3346bytes) at 1732670694799Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671 so closing down at 1732670694804 (+5 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000003-wal.1732670694671.temp at 1732670694806 (+2 ms)3 split writer threads finished at 1732670694806Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000003-wal.1732670694671.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732670694813 (+7 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000003-wal.1732670694671.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000035 at 1732670694814 (+1 ms)Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671, size=3.3 K, length=3346, corrupted=false, cancelled=false at 1732670694815 (+1 ms) 2024-11-27T01:24:54,816 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694671 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670694671 2024-11-27T01:24:54,817 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000035 2024-11-27T01:24:54,818 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:54,819 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:54,833 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820, exclude list is [], retry=0 2024-11-27T01:24:54,835 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:54,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:54,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:54,838 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 2024-11-27T01:24:54,839 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32867:32867),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:44901:44901)] 2024-11-27T01:24:54,839 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bfa43c595f131a8070100a905a27a0b8, NAME => 'testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:24:54,839 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:54,839 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,839 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,840 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,841 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName a 2024-11-27T01:24:54,841 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,846 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/a/a1724476fedb4978b9129e5a78981e82 2024-11-27T01:24:54,846 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,846 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,847 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName b 2024-11-27T01:24:54,847 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,848 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,848 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,848 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName c 2024-11-27T01:24:54,848 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:54,849 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:54,849 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,849 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,851 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,851 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000035 2024-11-27T01:24:54,853 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:54,854 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000035 2024-11-27T01:24:54,855 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfa43c595f131a8070100a905a27a0b8 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-27T01:24:54,869 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/b/eecb2304c1ee4e949ccefbd248b10039 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732670694754/Put/seqid=0 2024-11-27T01:24:54,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741931_1110 (size=5958) 2024-11-27T01:24:54,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741931_1110 (size=5958) 2024-11-27T01:24:54,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741931_1110 (size=5958) 2024-11-27T01:24:54,877 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/b/eecb2304c1ee4e949ccefbd248b10039 2024-11-27T01:24:54,896 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/c/4f9258ff3ab34d30afea919b091dff69 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732670694762/Put/seqid=0 2024-11-27T01:24:54,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741932_1111 (size=5958) 2024-11-27T01:24:54,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741932_1111 (size=5958) 2024-11-27T01:24:54,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741932_1111 (size=5958) 2024-11-27T01:24:54,902 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/c/4f9258ff3ab34d30afea919b091dff69 2024-11-27T01:24:54,907 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/b/eecb2304c1ee4e949ccefbd248b10039 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/b/eecb2304c1ee4e949ccefbd248b10039 2024-11-27T01:24:54,912 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/b/eecb2304c1ee4e949ccefbd248b10039, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T01:24:54,913 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/c/4f9258ff3ab34d30afea919b091dff69 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/c/4f9258ff3ab34d30afea919b091dff69 2024-11-27T01:24:54,918 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/c/4f9258ff3ab34d30afea919b091dff69, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T01:24:54,918 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for bfa43c595f131a8070100a905a27a0b8 in 64ms, sequenceid=35, compaction requested=false; wal=null 2024-11-27T01:24:54,919 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000035 2024-11-27T01:24:54,920 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,920 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,920 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:54,922 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:54,924 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-27T01:24:54,925 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfa43c595f131a8070100a905a27a0b8; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69859842, jitterRate=0.04099276661872864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:54,925 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfa43c595f131a8070100a905a27a0b8: Writing region info on filesystem at 1732670694839Initializing all the Stores at 1732670694840 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694840Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694840Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670694840Obtaining lock to block concurrent updates at 1732670694855 (+15 ms)Preparing flush snapshotting stores in bfa43c595f131a8070100a905a27a0b8 at 1732670694855Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732670694855Flushing stores of testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. at 1732670694855Flushing bfa43c595f131a8070100a905a27a0b8/b: creating writer at 1732670694855Flushing bfa43c595f131a8070100a905a27a0b8/b: appending metadata at 1732670694869 (+14 ms)Flushing bfa43c595f131a8070100a905a27a0b8/b: closing flushed file at 1732670694869Flushing bfa43c595f131a8070100a905a27a0b8/c: creating writer at 1732670694882 (+13 ms)Flushing bfa43c595f131a8070100a905a27a0b8/c: appending metadata at 1732670694895 (+13 ms)Flushing bfa43c595f131a8070100a905a27a0b8/c: closing flushed file at 1732670694895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36dd6e49: reopening flushed file at 1732670694906 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@530d502f: reopening flushed file at 1732670694912 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for bfa43c595f131a8070100a905a27a0b8 in 64ms, sequenceid=35, compaction requested=false; wal=null at 1732670694918 (+6 ms)Cleaning up temporary data from old regions at 1732670694920 (+2 ms)Region opened successfully at 1732670694925 (+5 ms) 2024-11-27T01:24:55,014 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820, size=0 (0bytes) 2024-11-27T01:24:55,014 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 might be still open, length is 0 2024-11-27T01:24:55,014 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 2024-11-27T01:24:55,015 WARN [IPC Server handler 4 on default port 41069 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-27T01:24:55,015 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 after 1ms 2024-11-27T01:24:57,755 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:50156 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36269:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50156 dst: /127.0.0.1:36269 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36269 remote=/127.0.0.1:50156]. Total timeout mills is 60000, 57205 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:57,755 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:52700 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45195:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52700 dst: /127.0.0.1:45195 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:57,756 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1964801773_22 at /127.0.0.1:56608 [Receiving block BP-272052841-172.17.0.2-1732670656425:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37205:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56608 dst: /127.0.0.1:37205 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:57,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741930_1112 (size=2936) 2024-11-27T01:24:57,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741930_1112 (size=2936) 2024-11-27T01:24:59,016 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 after 4002ms 2024-11-27T01:24:59,019 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:59,019 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 took 4005ms 2024-11-27T01:24:59,021 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820; continuing. 2024-11-27T01:24:59,021 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 so closing down 2024-11-27T01:24:59,021 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:24:59,023 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732670694820.temp 2024-11-27T01:24:59,024 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000037-wal.1732670694820.temp 2024-11-27T01:24:59,025 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:24:59,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741933_1113 (size=2944) 2024-11-27T01:24:59,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741933_1113 (size=2944) 2024-11-27T01:24:59,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741933_1113 (size=2944) 2024-11-27T01:24:59,032 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000037-wal.1732670694820.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T01:24:59,033 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000037-wal.1732670694820.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000066 2024-11-27T01:24:59,033 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:24:59,033 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820, journal: Splitting hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820, size=0 (0bytes) at 1732670695014Finishing writing output for hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 so closing down at 1732670699021 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000037-wal.1732670694820.temp at 1732670699024 (+3 ms)3 split writer threads finished at 1732670699025 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000037-wal.1732670694820.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732670699032 (+7 ms)Rename recovered edits hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000037-wal.1732670694820.temp to hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000066 at 1732670699033 (+1 ms)Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820, size=0, length=0, corrupted=false, cancelled=false at 1732670699033 2024-11-27T01:24:59,035 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 to hdfs://localhost:41069/hbase/oldWALs/wal.1732670694820 2024-11-27T01:24:59,036 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000066 2024-11-27T01:24:59,036 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:24:59,037 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41069/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513, archiveDir=hdfs://localhost:41069/hbase/oldWALs, maxLogs=32 2024-11-27T01:24:59,051 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670699038, exclude list is [], retry=0 2024-11-27T01:24:59,053 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37205,DS-42f495a5-2548-46b1-8b0e-84e368707ff8,DISK] 2024-11-27T01:24:59,054 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45195,DS-ab87a46f-5473-416a-b73c-24835b0a27a4,DISK] 2024-11-27T01:24:59,054 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36269,DS-9801fc90-1be0-4920-be50-0b39128a6585,DISK] 2024-11-27T01:24:59,055 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670699038 2024-11-27T01:24:59,056 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44901:44901),(127.0.0.1/127.0.0.1:42585:42585),(127.0.0.1/127.0.0.1:32867:32867)] 2024-11-27T01:24:59,056 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:24:59,057 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,058 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName a 2024-11-27T01:24:59,058 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:59,063 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/a/a1724476fedb4978b9129e5a78981e82 2024-11-27T01:24:59,063 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:59,063 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,064 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName b 2024-11-27T01:24:59,064 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:59,068 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/b/eecb2304c1ee4e949ccefbd248b10039 2024-11-27T01:24:59,068 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:59,068 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,069 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa43c595f131a8070100a905a27a0b8 columnFamilyName c 2024-11-27T01:24:59,069 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:24:59,073 DEBUG [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/c/4f9258ff3ab34d30afea919b091dff69 2024-11-27T01:24:59,073 INFO [StoreOpener-bfa43c595f131a8070100a905a27a0b8-1 {}] regionserver.HStore(327): Store=bfa43c595f131a8070100a905a27a0b8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:24:59,073 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,074 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,075 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,076 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000066 2024-11-27T01:24:59,077 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-27T01:24:59,082 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000066 2024-11-27T01:24:59,083 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfa43c595f131a8070100a905a27a0b8 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T01:24:59,098 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/a/4c9b735c08124936935d91d77d10fb89 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732670694932/Put/seqid=0 2024-11-27T01:24:59,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741935_1115 (size=5958) 2024-11-27T01:24:59,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741935_1115 (size=5958) 2024-11-27T01:24:59,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741935_1115 (size=5958) 2024-11-27T01:24:59,104 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/a/4c9b735c08124936935d91d77d10fb89 2024-11-27T01:24:59,128 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/b/0360684049424db8ba40dcef1bb9f1d4 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732670694940/Put/seqid=0 2024-11-27T01:24:59,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741936_1116 (size=5958) 2024-11-27T01:24:59,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741936_1116 (size=5958) 2024-11-27T01:24:59,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741936_1116 (size=5958) 2024-11-27T01:24:59,135 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/b/0360684049424db8ba40dcef1bb9f1d4 2024-11-27T01:24:59,159 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/c/bdd0e7d0b23d4244aa199a9b953b09eb is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732670694952/Put/seqid=0 2024-11-27T01:24:59,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741937_1117 (size=5958) 2024-11-27T01:24:59,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741937_1117 (size=5958) 2024-11-27T01:24:59,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741937_1117 (size=5958) 2024-11-27T01:24:59,166 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/c/bdd0e7d0b23d4244aa199a9b953b09eb 2024-11-27T01:24:59,170 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/a/4c9b735c08124936935d91d77d10fb89 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/a/4c9b735c08124936935d91d77d10fb89 2024-11-27T01:24:59,174 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/a/4c9b735c08124936935d91d77d10fb89, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T01:24:59,175 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/b/0360684049424db8ba40dcef1bb9f1d4 as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/b/0360684049424db8ba40dcef1bb9f1d4 2024-11-27T01:24:59,179 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/b/0360684049424db8ba40dcef1bb9f1d4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T01:24:59,179 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/.tmp/c/bdd0e7d0b23d4244aa199a9b953b09eb as hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/c/bdd0e7d0b23d4244aa199a9b953b09eb 2024-11-27T01:24:59,183 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/c/bdd0e7d0b23d4244aa199a9b953b09eb, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T01:24:59,184 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for bfa43c595f131a8070100a905a27a0b8 in 101ms, sequenceid=66, compaction requested=false; wal=null 2024-11-27T01:24:59,184 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/0000000000000000066 2024-11-27T01:24:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,186 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:24:59,187 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bfa43c595f131a8070100a905a27a0b8 2024-11-27T01:24:59,188 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/hbase/data/default/testReplayEditsWrittenViaHRegion/bfa43c595f131a8070100a905a27a0b8/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-27T01:24:59,189 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bfa43c595f131a8070100a905a27a0b8; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71445104, jitterRate=0.06461501121520996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:24:59,189 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bfa43c595f131a8070100a905a27a0b8: Writing region info on filesystem at 1732670699056Initializing all the Stores at 1732670699057 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670699057Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670699057Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670699057Obtaining lock to block concurrent updates at 1732670699083 (+26 ms)Preparing flush snapshotting stores in bfa43c595f131a8070100a905a27a0b8 at 1732670699083Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732670699083Flushing stores of testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. at 1732670699083Flushing bfa43c595f131a8070100a905a27a0b8/a: creating writer at 1732670699083Flushing bfa43c595f131a8070100a905a27a0b8/a: appending metadata at 1732670699097 (+14 ms)Flushing bfa43c595f131a8070100a905a27a0b8/a: closing flushed file at 1732670699097Flushing bfa43c595f131a8070100a905a27a0b8/b: creating writer at 1732670699109 (+12 ms)Flushing bfa43c595f131a8070100a905a27a0b8/b: appending metadata at 1732670699127 (+18 ms)Flushing bfa43c595f131a8070100a905a27a0b8/b: closing flushed file at 1732670699127Flushing bfa43c595f131a8070100a905a27a0b8/c: creating writer at 1732670699140 (+13 ms)Flushing bfa43c595f131a8070100a905a27a0b8/c: appending metadata at 1732670699159 (+19 ms)Flushing bfa43c595f131a8070100a905a27a0b8/c: closing flushed file at 1732670699159Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68907652: reopening flushed file at 1732670699170 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23f9c332: reopening flushed file at 1732670699174 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cc9ec0d: reopening flushed file at 1732670699179 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for bfa43c595f131a8070100a905a27a0b8 in 101ms, sequenceid=66, compaction requested=false; wal=null at 1732670699184 (+5 ms)Cleaning up temporary data from old regions at 1732670699185 (+1 ms)Region opened successfully at 1732670699189 (+4 ms) 2024-11-27T01:24:59,201 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bfa43c595f131a8070100a905a27a0b8, disabling compactions & flushes 2024-11-27T01:24:59,201 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:59,201 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:59,201 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. after waiting 0 ms 2024-11-27T01:24:59,201 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:59,203 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670694514.bfa43c595f131a8070100a905a27a0b8. 2024-11-27T01:24:59,203 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bfa43c595f131a8070100a905a27a0b8: Waiting for close lock at 1732670699201Disabling compacts and flushes for region at 1732670699201Disabling writes for close at 1732670699201Writing region close event to WAL at 1732670699202 (+1 ms)Closed at 1732670699203 (+1 ms) 2024-11-27T01:24:59,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741934_1114 (size=95) 2024-11-27T01:24:59,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741934_1114 (size=95) 2024-11-27T01:24:59,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741934_1114 (size=95) 2024-11-27T01:24:59,207 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:24:59,207 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670699038) 2024-11-27T01:24:59,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=443 (was 436) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_595008448_22 at /127.0.0.1:52750 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1774832404) connection to localhost/127.0.0.1:41069 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_595008448_22 at /127.0.0.1:50220 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_595008448_22 at /127.0.0.1:56634 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:41069 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=211 (was 221), ProcessCount=11 (was 11), AvailableMemoryMB=5985 (was 6023) 2024-11-27T01:24:59,221 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-11-27T01:24:59,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T01:24:59,222 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:24:59,222 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:24:59,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:59,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:59,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:24:59,223 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T01:24:59,223 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=207465779, stopped=false 2024-11-27T01:24:59,223 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7d6f26a4c136,43429,1732670659874 2024-11-27T01:24:59,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:24:59,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:24:59,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:59,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:24:59,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:59,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:59,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:24:59,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:24:59,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:24:59,227 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:24:59,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:24:59,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:59,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:24:59,227 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,34199,1732670660585' ***** 2024-11-27T01:24:59,227 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T01:24:59,227 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,39619,1732670660728' ***** 2024-11-27T01:24:59,227 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T01:24:59,228 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:24:59,228 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:24:59,228 INFO [RS:2;7d6f26a4c136:39619 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T01:24:59,228 INFO [RS:2;7d6f26a4c136:39619 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T01:24:59,228 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(959): stopping server 7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:59,228 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:24:59,228 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:24:59,228 INFO [RS:0;7d6f26a4c136:34199 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T01:24:59,228 INFO [RS:2;7d6f26a4c136:39619 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7d6f26a4c136:39619. 2024-11-27T01:24:59,228 INFO [RS:0;7d6f26a4c136:34199 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T01:24:59,228 DEBUG [RS:2;7d6f26a4c136:39619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:24:59,228 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(3091): Received CLOSE for 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:59,228 DEBUG [RS:2;7d6f26a4c136:39619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:59,229 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(959): stopping server 7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:59,229 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:24:59,229 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:24:59,229 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:24:59,229 INFO [RS:0;7d6f26a4c136:34199 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7d6f26a4c136:34199. 2024-11-27T01:24:59,229 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:24:59,229 DEBUG [RS:0;7d6f26a4c136:34199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:24:59,229 DEBUG [RS:0;7d6f26a4c136:34199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:59,229 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T01:24:59,229 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 159a502e808ebaf9e21757c52ae64154, disabling compactions & flushes 2024-11-27T01:24:59,229 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T01:24:59,229 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:59,229 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1325): Online Regions={159a502e808ebaf9e21757c52ae64154=testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154.} 2024-11-27T01:24:59,229 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:59,229 DEBUG [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1351): Waiting on 159a502e808ebaf9e21757c52ae64154 2024-11-27T01:24:59,229 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. after waiting 0 ms 2024-11-27T01:24:59,229 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:59,229 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:24:59,230 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T01:24:59,230 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T01:24:59,230 DEBUG [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T01:24:59,230 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:24:59,230 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:24:59,230 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:24:59,230 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:24:59,230 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:24:59,230 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-27T01:24:59,235 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/default/testReplayEditsAfterRegionMovedWithMultiCF/159a502e808ebaf9e21757c52ae64154/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-27T01:24:59,236 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:59,236 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 159a502e808ebaf9e21757c52ae64154: Waiting for close lock at 1732670699229Running coprocessor pre-close hooks at 1732670699229Disabling compacts and flushes for region at 1732670699229Disabling writes for close at 1732670699229Writing region close event to WAL at 1732670699230 (+1 ms)Running coprocessor post-close hooks at 1732670699236 (+6 ms)Closed at 1732670699236 2024-11-27T01:24:59,236 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154. 2024-11-27T01:24:59,247 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:24:59,255 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:24:59,256 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/info/1e945cebc6b8484db05d0fb149f7ecf7 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732670675623.159a502e808ebaf9e21757c52ae64154./info:regioninfo/1732670678666/Put/seqid=0 2024-11-27T01:24:59,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741938_1118 (size=8243) 2024-11-27T01:24:59,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741938_1118 (size=8243) 2024-11-27T01:24:59,265 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/info/1e945cebc6b8484db05d0fb149f7ecf7 2024-11-27T01:24:59,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741938_1118 (size=8243) 2024-11-27T01:24:59,286 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/ns/097d23ac7c7945d6813b9bb0f28d476a is 43, key is default/ns:d/1732670662733/Put/seqid=0 2024-11-27T01:24:59,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741939_1119 (size=5153) 2024-11-27T01:24:59,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741939_1119 (size=5153) 2024-11-27T01:24:59,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741939_1119 (size=5153) 2024-11-27T01:24:59,293 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/ns/097d23ac7c7945d6813b9bb0f28d476a 2024-11-27T01:24:59,320 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/table/d7d117f895364060acc90222745e86f9 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732670676050/Put/seqid=0 2024-11-27T01:24:59,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741940_1120 (size=5431) 2024-11-27T01:24:59,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741940_1120 (size=5431) 2024-11-27T01:24:59,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741940_1120 (size=5431) 2024-11-27T01:24:59,327 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/table/d7d117f895364060acc90222745e86f9 2024-11-27T01:24:59,333 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/info/1e945cebc6b8484db05d0fb149f7ecf7 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/info/1e945cebc6b8484db05d0fb149f7ecf7 2024-11-27T01:24:59,339 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/info/1e945cebc6b8484db05d0fb149f7ecf7, entries=18, sequenceid=21, filesize=8.0 K 2024-11-27T01:24:59,340 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/ns/097d23ac7c7945d6813b9bb0f28d476a as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/ns/097d23ac7c7945d6813b9bb0f28d476a 2024-11-27T01:24:59,345 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/ns/097d23ac7c7945d6813b9bb0f28d476a, entries=2, sequenceid=21, filesize=5.0 K 2024-11-27T01:24:59,346 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/.tmp/table/d7d117f895364060acc90222745e86f9 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/table/d7d117f895364060acc90222745e86f9 2024-11-27T01:24:59,351 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/table/d7d117f895364060acc90222745e86f9, entries=2, sequenceid=21, filesize=5.3 K 2024-11-27T01:24:59,352 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=21, compaction requested=false 2024-11-27T01:24:59,356 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-27T01:24:59,357 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:24:59,357 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:24:59,357 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732670699230Running coprocessor pre-close hooks at 1732670699230Disabling compacts and flushes for region at 1732670699230Disabling writes for close at 1732670699230Obtaining lock to block concurrent updates at 1732670699230Preparing flush snapshotting stores in 1588230740 at 1732670699230Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732670699231 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732670699232 (+1 ms)Flushing 1588230740/info: creating writer at 1732670699232Flushing 1588230740/info: appending metadata at 1732670699256 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732670699256Flushing 1588230740/ns: creating writer at 1732670699271 (+15 ms)Flushing 1588230740/ns: appending metadata at 1732670699285 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732670699285Flushing 1588230740/table: creating writer at 1732670699298 (+13 ms)Flushing 1588230740/table: appending metadata at 1732670699319 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732670699319Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3475f2d2: reopening flushed file at 1732670699332 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e562869: reopening flushed file at 1732670699340 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7469e85: reopening flushed file at 1732670699346 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=21, compaction requested=false at 1732670699352 (+6 ms)Writing region close event to WAL at 1732670699353 (+1 ms)Running coprocessor post-close hooks at 1732670699357 (+4 ms)Closed at 1732670699357 2024-11-27T01:24:59,357 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T01:24:59,429 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,34199,1732670660585; all regions closed. 2024-11-27T01:24:59,430 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,39619,1732670660728; all regions closed. 2024-11-27T01:24:59,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741833_1009 (size=2187) 2024-11-27T01:24:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741833_1009 (size=2187) 2024-11-27T01:24:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741836_1012 (size=4718) 2024-11-27T01:24:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741833_1009 (size=2187) 2024-11-27T01:24:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741836_1012 (size=4718) 2024-11-27T01:24:59,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741836_1012 (size=4718) 2024-11-27T01:24:59,435 DEBUG [RS:2;7d6f26a4c136:39619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs 2024-11-27T01:24:59,435 INFO [RS:2;7d6f26a4c136:39619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C39619%2C1732670660728.meta:.meta(num 1732670662543) 2024-11-27T01:24:59,435 DEBUG [RS:0;7d6f26a4c136:34199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs 2024-11-27T01:24:59,435 INFO [RS:0;7d6f26a4c136:34199 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C34199%2C1732670660585:(num 1732670662106) 2024-11-27T01:24:59,436 DEBUG [RS:0;7d6f26a4c136:34199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:59,436 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:24:59,436 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:24:59,436 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-27T01:24:59,436 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:24:59,436 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:24:59,436 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:24:59,436 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:24:59,436 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:24:59,437 INFO [RS:0;7d6f26a4c136:34199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34199 2024-11-27T01:24:59,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741834_1010 (size=95) 2024-11-27T01:24:59,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741834_1010 (size=95) 2024-11-27T01:24:59,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741834_1010 (size=95) 2024-11-27T01:24:59,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,34199,1732670660585 2024-11-27T01:24:59,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:24:59,439 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:24:59,441 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,34199,1732670660585] 2024-11-27T01:24:59,441 DEBUG [RS:2;7d6f26a4c136:39619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/oldWALs 2024-11-27T01:24:59,441 INFO [RS:2;7d6f26a4c136:39619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C39619%2C1732670660728:(num 1732670662103) 2024-11-27T01:24:59,441 DEBUG [RS:2;7d6f26a4c136:39619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:24:59,441 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:24:59,441 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:24:59,441 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:24:59,442 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:24:59,442 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:24:59,442 INFO [RS:2;7d6f26a4c136:39619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39619 2024-11-27T01:24:59,442 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,34199,1732670660585 already deleted, retry=false 2024-11-27T01:24:59,443 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d6f26a4c136,34199,1732670660585 expired; onlineServers=1 2024-11-27T01:24:59,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,39619,1732670660728 2024-11-27T01:24:59,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:24:59,445 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:24:59,446 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,39619,1732670660728] 2024-11-27T01:24:59,447 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,39619,1732670660728 already deleted, retry=false 2024-11-27T01:24:59,447 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d6f26a4c136,39619,1732670660728 expired; onlineServers=0 2024-11-27T01:24:59,448 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7d6f26a4c136,43429,1732670659874' ***** 2024-11-27T01:24:59,448 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T01:24:59,448 INFO [M:0;7d6f26a4c136:43429 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:24:59,448 INFO [M:0;7d6f26a4c136:43429 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:24:59,448 DEBUG [M:0;7d6f26a4c136:43429 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T01:24:59,448 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T01:24:59,448 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670661767 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670661767,5,FailOnTimeoutGroup] 2024-11-27T01:24:59,448 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670661768 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670661768,5,FailOnTimeoutGroup] 2024-11-27T01:24:59,448 DEBUG [M:0;7d6f26a4c136:43429 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T01:24:59,448 INFO [M:0;7d6f26a4c136:43429 {}] hbase.ChoreService(370): Chore service for: master/7d6f26a4c136:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T01:24:59,449 INFO [M:0;7d6f26a4c136:43429 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:24:59,449 DEBUG [M:0;7d6f26a4c136:43429 {}] master.HMaster(1795): Stopping service threads 2024-11-27T01:24:59,449 INFO [M:0;7d6f26a4c136:43429 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T01:24:59,449 INFO [M:0;7d6f26a4c136:43429 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:24:59,450 INFO [M:0;7d6f26a4c136:43429 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:24:59,450 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T01:24:59,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T01:24:59,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:24:59,450 DEBUG [M:0;7d6f26a4c136:43429 {}] zookeeper.ZKUtil(347): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T01:24:59,450 WARN [M:0;7d6f26a4c136:43429 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T01:24:59,451 INFO [M:0;7d6f26a4c136:43429 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/.lastflushedseqids 2024-11-27T01:24:59,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741941_1121 (size=138) 2024-11-27T01:24:59,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741941_1121 (size=138) 2024-11-27T01:24:59,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741941_1121 (size=138) 2024-11-27T01:24:59,464 INFO [M:0;7d6f26a4c136:43429 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:24:59,464 INFO [M:0;7d6f26a4c136:43429 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T01:24:59,464 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:24:59,464 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:59,464 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:59,464 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:24:59,464 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:59,464 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-11-27T01:24:59,480 DEBUG [M:0;7d6f26a4c136:43429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d91053b6ecb04c729cc25a8fb10de888 is 82, key is hbase:meta,,1/info:regioninfo/1732670662625/Put/seqid=0 2024-11-27T01:24:59,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741942_1122 (size=5672) 2024-11-27T01:24:59,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741942_1122 (size=5672) 2024-11-27T01:24:59,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741942_1122 (size=5672) 2024-11-27T01:24:59,486 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d91053b6ecb04c729cc25a8fb10de888 2024-11-27T01:24:59,504 DEBUG [M:0;7d6f26a4c136:43429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/977c9592685449f9a9b4f54c228b22a1 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732670676056/Put/seqid=0 2024-11-27T01:24:59,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741943_1123 (size=7756) 2024-11-27T01:24:59,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741943_1123 (size=7756) 2024-11-27T01:24:59,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741943_1123 (size=7756) 2024-11-27T01:24:59,516 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/977c9592685449f9a9b4f54c228b22a1 2024-11-27T01:24:59,521 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 977c9592685449f9a9b4f54c228b22a1 2024-11-27T01:24:59,537 DEBUG [M:0;7d6f26a4c136:43429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff51552bb89d464ebe5bcffd3ca6c6d0 is 69, key is 7d6f26a4c136,34199,1732670660585/rs:state/1732670661800/Put/seqid=0 2024-11-27T01:24:59,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:59,541 INFO [RS:0;7d6f26a4c136:34199 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:24:59,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34199-0x1015706048a0001, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:59,541 INFO [RS:0;7d6f26a4c136:34199 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,34199,1732670660585; zookeeper connection closed. 2024-11-27T01:24:59,541 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9553af9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9553af9 2024-11-27T01:24:59,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741944_1124 (size=5440) 2024-11-27T01:24:59,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741944_1124 (size=5440) 2024-11-27T01:24:59,546 INFO [RS:2;7d6f26a4c136:39619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:24:59,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:59,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741944_1124 (size=5440) 2024-11-27T01:24:59,546 INFO [RS:2;7d6f26a4c136:39619 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,39619,1732670660728; zookeeper connection closed. 2024-11-27T01:24:59,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39619-0x1015706048a0003, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:59,547 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4fd0b51e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4fd0b51e 2024-11-27T01:24:59,547 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T01:24:59,550 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff51552bb89d464ebe5bcffd3ca6c6d0 2024-11-27T01:24:59,555 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ff51552bb89d464ebe5bcffd3ca6c6d0 2024-11-27T01:24:59,556 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d91053b6ecb04c729cc25a8fb10de888 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d91053b6ecb04c729cc25a8fb10de888 2024-11-27T01:24:59,561 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d91053b6ecb04c729cc25a8fb10de888, entries=8, sequenceid=168, filesize=5.5 K 2024-11-27T01:24:59,562 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/977c9592685449f9a9b4f54c228b22a1 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/977c9592685449f9a9b4f54c228b22a1 2024-11-27T01:24:59,566 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 977c9592685449f9a9b4f54c228b22a1 2024-11-27T01:24:59,567 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/977c9592685449f9a9b4f54c228b22a1, entries=17, sequenceid=168, filesize=7.6 K 2024-11-27T01:24:59,567 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ff51552bb89d464ebe5bcffd3ca6c6d0 as hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ff51552bb89d464ebe5bcffd3ca6c6d0 2024-11-27T01:24:59,572 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ff51552bb89d464ebe5bcffd3ca6c6d0 2024-11-27T01:24:59,572 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41069/user/jenkins/test-data/95b8acca-45d7-251b-2cf0-4b23b9fc0b73/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ff51552bb89d464ebe5bcffd3ca6c6d0, entries=3, sequenceid=168, filesize=5.3 K 2024-11-27T01:24:59,573 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=168, compaction requested=false 2024-11-27T01:24:59,574 INFO [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:24:59,574 DEBUG [M:0;7d6f26a4c136:43429 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732670699464Disabling compacts and flushes for region at 1732670699464Disabling writes for close at 1732670699464Obtaining lock to block concurrent updates at 1732670699464Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732670699464Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1732670699465 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732670699465Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732670699465Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732670699479 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732670699479Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732670699490 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732670699504 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732670699504Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732670699521 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732670699536 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732670699536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f2f4483: reopening flushed file at 1732670699555 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f59b0e: reopening flushed file at 1732670699561 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7170196d: reopening flushed file at 1732670699567 (+6 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=168, compaction requested=false at 1732670699573 (+6 ms)Writing region close event to WAL at 1732670699574 (+1 ms)Closed at 1732670699574 2024-11-27T01:24:59,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36269 is added to blk_1073741830_1006 (size=56528) 2024-11-27T01:24:59,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37205 is added to blk_1073741830_1006 (size=56528) 2024-11-27T01:24:59,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45195 is added to blk_1073741830_1006 (size=56528) 2024-11-27T01:24:59,578 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:24:59,578 INFO [M:0;7d6f26a4c136:43429 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T01:24:59,579 INFO [M:0;7d6f26a4c136:43429 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43429 2024-11-27T01:24:59,579 INFO [M:0;7d6f26a4c136:43429 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:24:59,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:59,683 INFO [M:0;7d6f26a4c136:43429 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:24:59,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43429-0x1015706048a0000, quorum=127.0.0.1:61959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:24:59,689 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679753 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679753 (inode 16655) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670679068/wal.1732670679753 (inode 16655) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:24:59,690 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670671119/wal.1732670671224 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:59,691 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670694513/wal.1732670694820 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:59,693 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670689164 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670689164 (inode 16678) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670679915/wal.1732670689164 (inode 16678) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:24:59,693 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670663931/wal.1732670663996 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:59,695 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689932 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689932 (inode 16704) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670689364/wal.1732670689932 (inode 16704) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:24:59,696 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670690094/wal.1732670690157 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:24:59,698 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal.1732670663752 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal.1732670663752 (inode 16485) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670663497/wal.1732670663752 (inode 16485) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:24:59,701 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal.1732670663240 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal.1732670663240 (inode 16462) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670662989/wal.1732670663240 (inode 16462) Holder DFSClient_NONMAPREDUCE_1964801773_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:24:59,706 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@47d9f377{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:24:59,708 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b0de5e3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:24:59,708 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:24:59,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f216234{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:24:59,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32dd0934{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,STOPPED} 2024-11-27T01:24:59,712 WARN [BP-272052841-172.17.0.2-1732670656425 heartbeating to localhost/127.0.0.1:41069 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:24:59,712 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:24:59,712 WARN [BP-272052841-172.17.0.2-1732670656425 heartbeating to localhost/127.0.0.1:41069 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-272052841-172.17.0.2-1732670656425 (Datanode Uuid 845fab92-3fee-4427-9122-1c2626f52414) service to localhost/127.0.0.1:41069 2024-11-27T01:24:59,712 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:24:59,714 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data5/current/BP-272052841-172.17.0.2-1732670656425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:24:59,714 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data6/current/BP-272052841-172.17.0.2-1732670656425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:24:59,715 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:24:59,718 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@80bbe1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:24:59,718 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14fd625f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:24:59,718 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:24:59,718 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7deb8c77{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:24:59,718 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4477f2ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,STOPPED} 2024-11-27T01:24:59,720 WARN [BP-272052841-172.17.0.2-1732670656425 heartbeating to localhost/127.0.0.1:41069 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:24:59,720 WARN [BP-272052841-172.17.0.2-1732670656425 heartbeating to localhost/127.0.0.1:41069 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-272052841-172.17.0.2-1732670656425 (Datanode Uuid 20838706-6b50-4ab4-8a33-a27afc0ef1e7) service to localhost/127.0.0.1:41069 2024-11-27T01:24:59,720 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:24:59,720 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:24:59,721 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data3/current/BP-272052841-172.17.0.2-1732670656425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:24:59,721 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data4/current/BP-272052841-172.17.0.2-1732670656425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:24:59,722 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:24:59,729 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d91a9f0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:24:59,730 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2ace2437{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:24:59,730 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:24:59,730 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fecd6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:24:59,730 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c12e3d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,STOPPED} 2024-11-27T01:24:59,732 WARN [BP-272052841-172.17.0.2-1732670656425 heartbeating to localhost/127.0.0.1:41069 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:24:59,732 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:24:59,732 WARN [BP-272052841-172.17.0.2-1732670656425 heartbeating to localhost/127.0.0.1:41069 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-272052841-172.17.0.2-1732670656425 (Datanode Uuid d4665d60-332c-462f-b070-91d408a09e0a) service to localhost/127.0.0.1:41069 2024-11-27T01:24:59,732 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:24:59,733 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data1/current/BP-272052841-172.17.0.2-1732670656425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:24:59,734 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/cluster_d4728812-1fb8-e081-dd33-0bf7183d7ccd/data/data2/current/BP-272052841-172.17.0.2-1732670656425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:24:59,734 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:24:59,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@126092e9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:24:59,742 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cf0cac{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:24:59,742 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:24:59,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1257581c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:24:59,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@139bf1d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/48687f83-a857-c504-6bb1-c698cb7fc53d/hadoop.log.dir/,STOPPED} 2024-11-27T01:24:59,755 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T01:24:59,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down