2024-11-26 15:28:51,565 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-26 15:28:51,575 main DEBUG Took 0.008850 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 15:28:51,576 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 15:28:51,576 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 15:28:51,577 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 15:28:51,578 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,585 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 15:28:51,595 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,597 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,598 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,599 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,599 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,600 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,601 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,601 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,602 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,603 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,603 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,603 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,604 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,604 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,604 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,605 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 15:28:51,605 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,606 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 15:28:51,607 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 15:28:51,608 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 15:28:51,610 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 15:28:51,610 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 15:28:51,612 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 15:28:51,612 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 15:28:51,620 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 15:28:51,622 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 15:28:51,624 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 15:28:51,624 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 15:28:51,625 main DEBUG createAppenders(={Console}) 2024-11-26 15:28:51,625 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-26 15:28:51,626 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-26 15:28:51,626 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-26 15:28:51,626 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 15:28:51,626 main DEBUG OutputStream closed 2024-11-26 15:28:51,627 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 15:28:51,627 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 15:28:51,627 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-26 15:28:51,695 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 15:28:51,697 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 15:28:51,698 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 15:28:51,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 15:28:51,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 15:28:51,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 15:28:51,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 15:28:51,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 15:28:51,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 15:28:51,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 15:28:51,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 15:28:51,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 15:28:51,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 15:28:51,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 15:28:51,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 15:28:51,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 15:28:51,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 15:28:51,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 15:28:51,705 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 15:28:51,705 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-26 15:28:51,706 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 15:28:51,706 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-26T15:28:51,919 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0 2024-11-26 15:28:51,922 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 15:28:51,922 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T15:28:51,931 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-26T15:28:51,956 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T15:28:51,998 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T15:28:51,998 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T15:28:52,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T15:28:52,031 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba, deleteOnExit=true 2024-11-26T15:28:52,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T15:28:52,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/test.cache.data in system properties and HBase conf 2024-11-26T15:28:52,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T15:28:52,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir in system properties and HBase conf 2024-11-26T15:28:52,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T15:28:52,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T15:28:52,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T15:28:52,115 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T15:28:52,208 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T15:28:52,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T15:28:52,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T15:28:52,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T15:28:52,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T15:28:52,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T15:28:52,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T15:28:52,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T15:28:52,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T15:28:52,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T15:28:52,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/nfs.dump.dir in system properties and HBase conf 2024-11-26T15:28:52,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/java.io.tmpdir in system properties and HBase conf 2024-11-26T15:28:52,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T15:28:52,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T15:28:52,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T15:28:53,510 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T15:28:53,580 INFO [Time-limited test {}] log.Log(170): Logging initialized @2671ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T15:28:53,648 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:28:53,703 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T15:28:53,726 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T15:28:53,726 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T15:28:53,727 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T15:28:53,745 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:28:53,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,AVAILABLE} 2024-11-26T15:28:53,750 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T15:28:53,953 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/java.io.tmpdir/jetty-localhost-45631-hadoop-hdfs-3_4_1-tests_jar-_-any-17080222450774444051/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T15:28:53,958 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:45631} 2024-11-26T15:28:53,958 INFO [Time-limited test {}] server.Server(415): Started @3049ms 2024-11-26T15:28:54,502 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:28:54,509 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T15:28:54,510 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T15:28:54,510 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T15:28:54,510 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T15:28:54,511 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,AVAILABLE} 2024-11-26T15:28:54,512 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T15:28:54,609 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/java.io.tmpdir/jetty-localhost-34083-hadoop-hdfs-3_4_1-tests_jar-_-any-6547748754920771401/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:28:54,610 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:34083} 2024-11-26T15:28:54,610 INFO [Time-limited test {}] server.Server(415): Started @3701ms 2024-11-26T15:28:54,655 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T15:28:54,764 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:28:54,769 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T15:28:54,772 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T15:28:54,773 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T15:28:54,773 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T15:28:54,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,AVAILABLE} 2024-11-26T15:28:54,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T15:28:54,877 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/java.io.tmpdir/jetty-localhost-38303-hadoop-hdfs-3_4_1-tests_jar-_-any-6822621629610183563/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:28:54,878 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:38303} 2024-11-26T15:28:54,878 INFO [Time-limited test {}] server.Server(415): Started @3969ms 2024-11-26T15:28:54,880 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T15:28:54,916 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T15:28:54,923 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T15:28:54,925 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T15:28:54,925 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T15:28:54,925 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T15:28:54,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,AVAILABLE} 2024-11-26T15:28:54,928 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T15:28:55,029 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/java.io.tmpdir/jetty-localhost-39095-hadoop-hdfs-3_4_1-tests_jar-_-any-1160077160777731025/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:28:55,030 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:39095} 2024-11-26T15:28:55,030 INFO [Time-limited test {}] server.Server(415): Started @4121ms 2024-11-26T15:28:55,032 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T15:28:56,541 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data4/current/BP-814321108-172.17.0.3-1732634932847/current, will proceed with Du for space computation calculation, 2024-11-26T15:28:56,541 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data3/current/BP-814321108-172.17.0.3-1732634932847/current, will proceed with Du for space computation calculation, 2024-11-26T15:28:56,541 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data1/current/BP-814321108-172.17.0.3-1732634932847/current, will proceed with Du for space computation calculation, 2024-11-26T15:28:56,541 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data2/current/BP-814321108-172.17.0.3-1732634932847/current, will proceed with Du for space computation calculation, 2024-11-26T15:28:56,576 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T15:28:56,576 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T15:28:56,594 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data5/current/BP-814321108-172.17.0.3-1732634932847/current, will proceed with Du for space computation calculation, 2024-11-26T15:28:56,595 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data6/current/BP-814321108-172.17.0.3-1732634932847/current, will proceed with Du for space computation calculation, 2024-11-26T15:28:56,616 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T15:28:56,624 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1810bd74cfb5e9c1 with lease ID 0xae93ab6fe4691781: Processing first storage report for DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624 from datanode DatanodeRegistration(127.0.0.1:34329, datanodeUuid=405b8008-fac7-42f3-91b5-9ae72de4f169, infoPort=45859, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847) 2024-11-26T15:28:56,625 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1810bd74cfb5e9c1 with lease ID 0xae93ab6fe4691781: from storage DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624 node DatanodeRegistration(127.0.0.1:34329, datanodeUuid=405b8008-fac7-42f3-91b5-9ae72de4f169, infoPort=45859, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T15:28:56,625 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50f55678be18ec55 with lease ID 0xae93ab6fe4691782: Processing first storage report for DS-aef6346b-d853-45b3-9671-8aa68d3dd674 from datanode DatanodeRegistration(127.0.0.1:39747, datanodeUuid=2110294a-95fc-49c4-bdc4-7f402fcbd37b, infoPort=39591, infoSecurePort=0, ipcPort=44377, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847) 2024-11-26T15:28:56,625 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50f55678be18ec55 with lease ID 0xae93ab6fe4691782: from storage DS-aef6346b-d853-45b3-9671-8aa68d3dd674 node DatanodeRegistration(127.0.0.1:39747, datanodeUuid=2110294a-95fc-49c4-bdc4-7f402fcbd37b, infoPort=39591, infoSecurePort=0, ipcPort=44377, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T15:28:56,625 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x627bcaaf27f63410 with lease ID 0xae93ab6fe4691783: Processing first storage report for DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3 from datanode DatanodeRegistration(127.0.0.1:35535, datanodeUuid=e94b488b-994a-402a-8fa1-f753723bc7c1, infoPort=36049, infoSecurePort=0, ipcPort=39965, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847) 2024-11-26T15:28:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x627bcaaf27f63410 with lease ID 0xae93ab6fe4691783: from storage DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3 node DatanodeRegistration(127.0.0.1:35535, datanodeUuid=e94b488b-994a-402a-8fa1-f753723bc7c1, infoPort=36049, infoSecurePort=0, ipcPort=39965, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T15:28:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1810bd74cfb5e9c1 with lease ID 0xae93ab6fe4691781: Processing first storage report for DS-ec1a5d12-15df-47df-a5ff-2c7334339743 from datanode DatanodeRegistration(127.0.0.1:34329, datanodeUuid=405b8008-fac7-42f3-91b5-9ae72de4f169, infoPort=45859, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847) 2024-11-26T15:28:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1810bd74cfb5e9c1 with lease ID 0xae93ab6fe4691781: from storage DS-ec1a5d12-15df-47df-a5ff-2c7334339743 node DatanodeRegistration(127.0.0.1:34329, datanodeUuid=405b8008-fac7-42f3-91b5-9ae72de4f169, infoPort=45859, infoSecurePort=0, ipcPort=37445, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T15:28:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x50f55678be18ec55 with lease ID 0xae93ab6fe4691782: Processing first storage report for DS-26df43c7-129b-4dc6-abcb-21a109ae7491 from datanode DatanodeRegistration(127.0.0.1:39747, datanodeUuid=2110294a-95fc-49c4-bdc4-7f402fcbd37b, infoPort=39591, infoSecurePort=0, ipcPort=44377, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847) 2024-11-26T15:28:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x50f55678be18ec55 with lease ID 0xae93ab6fe4691782: from storage DS-26df43c7-129b-4dc6-abcb-21a109ae7491 node DatanodeRegistration(127.0.0.1:39747, datanodeUuid=2110294a-95fc-49c4-bdc4-7f402fcbd37b, infoPort=39591, infoSecurePort=0, ipcPort=44377, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T15:28:56,626 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x627bcaaf27f63410 with lease ID 0xae93ab6fe4691783: Processing first storage report for DS-caad07e4-83f9-4681-83ff-04a1ca633af7 from datanode DatanodeRegistration(127.0.0.1:35535, datanodeUuid=e94b488b-994a-402a-8fa1-f753723bc7c1, infoPort=36049, infoSecurePort=0, ipcPort=39965, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847) 2024-11-26T15:28:56,627 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x627bcaaf27f63410 with lease ID 0xae93ab6fe4691783: from storage DS-caad07e4-83f9-4681-83ff-04a1ca633af7 node DatanodeRegistration(127.0.0.1:35535, datanodeUuid=e94b488b-994a-402a-8fa1-f753723bc7c1, infoPort=36049, infoSecurePort=0, ipcPort=39965, storageInfo=lv=-57;cid=testClusterID;nsid=656269367;c=1732634932847), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T15:28:56,672 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0 2024-11-26T15:28:56,734 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/zookeeper_0, clientPort=60921, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T15:28:56,754 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60921 2024-11-26T15:28:56,768 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:56,771 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:56,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741825_1001 (size=7) 2024-11-26T15:28:56,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741825_1001 (size=7) 2024-11-26T15:28:56,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741825_1001 (size=7) 2024-11-26T15:28:57,366 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7 with version=8 2024-11-26T15:28:57,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/hbase-staging 2024-11-26T15:28:57,630 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6cc2bc8bf733:0 server-side Connection retries=45 2024-11-26T15:28:57,638 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:57,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:57,643 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T15:28:57,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:57,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T15:28:57,765 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T15:28:57,819 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T15:28:57,828 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T15:28:57,831 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T15:28:57,857 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 75849 (auto-detected) 2024-11-26T15:28:57,858 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-26T15:28:57,877 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44737 2024-11-26T15:28:57,896 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44737 connecting to ZooKeeper ensemble=127.0.0.1:60921 2024-11-26T15:28:58,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447370x0, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T15:28:58,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44737-0x10177c301390000 connected 2024-11-26T15:28:58,159 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,163 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:28:58,179 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7, hbase.cluster.distributed=false 2024-11-26T15:28:58,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T15:28:58,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44737 2024-11-26T15:28:58,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44737 2024-11-26T15:28:58,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44737 2024-11-26T15:28:58,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44737 2024-11-26T15:28:58,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44737 2024-11-26T15:28:58,296 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6cc2bc8bf733:0 server-side Connection retries=45 2024-11-26T15:28:58,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,298 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T15:28:58,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T15:28:58,300 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T15:28:58,303 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T15:28:58,304 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43161 2024-11-26T15:28:58,305 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43161 connecting to ZooKeeper ensemble=127.0.0.1:60921 2024-11-26T15:28:58,306 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,308 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:431610x0, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T15:28:58,317 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:431610x0, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:28:58,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43161-0x10177c301390001 connected 2024-11-26T15:28:58,320 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T15:28:58,328 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T15:28:58,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T15:28:58,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T15:28:58,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43161 2024-11-26T15:28:58,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43161 2024-11-26T15:28:58,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43161 2024-11-26T15:28:58,337 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43161 2024-11-26T15:28:58,337 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43161 2024-11-26T15:28:58,354 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6cc2bc8bf733:0 server-side Connection retries=45 2024-11-26T15:28:58,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,355 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T15:28:58,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T15:28:58,355 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T15:28:58,356 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T15:28:58,357 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41249 2024-11-26T15:28:58,358 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41249 connecting to ZooKeeper ensemble=127.0.0.1:60921 2024-11-26T15:28:58,359 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,361 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412490x0, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T15:28:58,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412490x0, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:28:58,369 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41249-0x10177c301390002 connected 2024-11-26T15:28:58,370 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T15:28:58,371 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T15:28:58,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T15:28:58,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T15:28:58,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41249 2024-11-26T15:28:58,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41249 2024-11-26T15:28:58,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41249 2024-11-26T15:28:58,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41249 2024-11-26T15:28:58,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41249 2024-11-26T15:28:58,395 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6cc2bc8bf733:0 server-side Connection retries=45 2024-11-26T15:28:58,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,396 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T15:28:58,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T15:28:58,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T15:28:58,396 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T15:28:58,396 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T15:28:58,397 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42689 2024-11-26T15:28:58,400 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42689 connecting to ZooKeeper ensemble=127.0.0.1:60921 2024-11-26T15:28:58,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,404 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426890x0, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T15:28:58,422 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:426890x0, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:28:58,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42689-0x10177c301390003 connected 2024-11-26T15:28:58,422 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T15:28:58,423 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T15:28:58,424 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T15:28:58,426 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T15:28:58,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42689 2024-11-26T15:28:58,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42689 2024-11-26T15:28:58,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42689 2024-11-26T15:28:58,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42689 2024-11-26T15:28:58,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42689 2024-11-26T15:28:58,446 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6cc2bc8bf733:44737 2024-11-26T15:28:58,447 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6cc2bc8bf733,44737,1732634937484 2024-11-26T15:28:58,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,466 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6cc2bc8bf733,44737,1732634937484 2024-11-26T15:28:58,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T15:28:58,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T15:28:58,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T15:28:58,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,496 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T15:28:58,497 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6cc2bc8bf733,44737,1732634937484 from backup master directory 2024-11-26T15:28:58,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6cc2bc8bf733,44737,1732634937484 2024-11-26T15:28:58,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T15:28:58,506 WARN [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T15:28:58,506 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6cc2bc8bf733,44737,1732634937484 2024-11-26T15:28:58,509 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T15:28:58,510 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T15:28:58,562 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/hbase.id] with ID: b86a04a6-48ba-44c0-ade9-bd78fee725c1 2024-11-26T15:28:58,563 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/.tmp/hbase.id 2024-11-26T15:28:58,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741826_1002 (size=42) 2024-11-26T15:28:58,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741826_1002 (size=42) 2024-11-26T15:28:58,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741826_1002 (size=42) 2024-11-26T15:28:58,578 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/.tmp/hbase.id]:[hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/hbase.id] 2024-11-26T15:28:58,623 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T15:28:58,627 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T15:28:58,644 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-26T15:28:58,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:58,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741827_1003 (size=196) 2024-11-26T15:28:58,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741827_1003 (size=196) 2024-11-26T15:28:58,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741827_1003 (size=196) 2024-11-26T15:28:58,689 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:28:58,691 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T15:28:58,696 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:28:58,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741828_1004 (size=1189) 2024-11-26T15:28:58,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741828_1004 (size=1189) 2024-11-26T15:28:58,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741828_1004 (size=1189) 2024-11-26T15:28:58,742 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store 2024-11-26T15:28:58,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741829_1005 (size=34) 2024-11-26T15:28:58,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741829_1005 (size=34) 2024-11-26T15:28:58,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741829_1005 (size=34) 2024-11-26T15:28:58,767 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T15:28:58,771 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:28:58,772 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T15:28:58,773 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:58,773 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:58,775 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T15:28:58,775 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:58,775 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:28:58,777 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732634938772Disabling compacts and flushes for region at 1732634938772Disabling writes for close at 1732634938775 (+3 ms)Writing region close event to WAL at 1732634938775Closed at 1732634938775 2024-11-26T15:28:58,779 WARN [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/.initializing 2024-11-26T15:28:58,779 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/WALs/6cc2bc8bf733,44737,1732634937484 2024-11-26T15:28:58,786 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:28:58,799 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C44737%2C1732634937484, suffix=, logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/WALs/6cc2bc8bf733,44737,1732634937484, archiveDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/oldWALs, maxLogs=10 2024-11-26T15:28:58,824 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/WALs/6cc2bc8bf733,44737,1732634937484/6cc2bc8bf733%2C44737%2C1732634937484.1732634938803, exclude list is [], retry=0 2024-11-26T15:28:58,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:28:58,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:28:58,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:28:58,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:28:58,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T15:28:58,883 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/WALs/6cc2bc8bf733,44737,1732634937484/6cc2bc8bf733%2C44737%2C1732634937484.1732634938803 2024-11-26T15:28:58,884 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:28:58,884 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:28:58,885 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:28:58,887 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,888 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,945 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T15:28:58,948 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:58,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:28:58,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,954 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T15:28:58,954 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:58,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:28:58,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T15:28:58,958 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:58,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:28:58,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T15:28:58,961 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:58,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:28:58,963 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,966 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,967 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,972 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,972 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,976 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T15:28:58,979 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T15:28:58,984 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:28:58,985 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69313684, jitterRate=0.03285437822341919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T15:28:58,993 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732634938899Initializing all the Stores at 1732634938901 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634938901Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634938902 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634938902Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634938902Cleaning up temporary data from old regions at 1732634938972 (+70 ms)Region opened successfully at 1732634938993 (+21 ms) 2024-11-26T15:28:58,995 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T15:28:59,026 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e1420e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6cc2bc8bf733/172.17.0.3:0 2024-11-26T15:28:59,052 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T15:28:59,061 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T15:28:59,061 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T15:28:59,063 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T15:28:59,065 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T15:28:59,069 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T15:28:59,069 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T15:28:59,091 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T15:28:59,099 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T15:28:59,147 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T15:28:59,149 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T15:28:59,151 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T15:28:59,158 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T15:28:59,159 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T15:28:59,163 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T15:28:59,168 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T15:28:59,169 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T15:28:59,179 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T15:28:59,199 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T15:28:59,210 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T15:28:59,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T15:28:59,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T15:28:59,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T15:28:59,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T15:28:59,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,225 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6cc2bc8bf733,44737,1732634937484, sessionid=0x10177c301390000, setting cluster-up flag (Was=false) 2024-11-26T15:28:59,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,285 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T15:28:59,290 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6cc2bc8bf733,44737,1732634937484 2024-11-26T15:28:59,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:28:59,348 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T15:28:59,353 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6cc2bc8bf733,44737,1732634937484 2024-11-26T15:28:59,361 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T15:28:59,428 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T15:28:59,435 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(746): ClusterId : b86a04a6-48ba-44c0-ade9-bd78fee725c1 2024-11-26T15:28:59,435 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(746): ClusterId : b86a04a6-48ba-44c0-ade9-bd78fee725c1 2024-11-26T15:28:59,435 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(746): ClusterId : b86a04a6-48ba-44c0-ade9-bd78fee725c1 2024-11-26T15:28:59,437 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T15:28:59,437 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T15:28:59,437 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T15:28:59,437 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T15:28:59,444 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T15:28:59,451 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6cc2bc8bf733,44737,1732634937484 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T15:28:59,454 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T15:28:59,454 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T15:28:59,454 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T15:28:59,454 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T15:28:59,454 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T15:28:59,454 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T15:28:59,475 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T15:28:59,475 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T15:28:59,475 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T15:28:59,475 DEBUG [RS:2;6cc2bc8bf733:42689 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71eed1c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6cc2bc8bf733/172.17.0.3:0 2024-11-26T15:28:59,475 DEBUG [RS:1;6cc2bc8bf733:41249 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fc29bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6cc2bc8bf733/172.17.0.3:0 2024-11-26T15:28:59,475 DEBUG [RS:0;6cc2bc8bf733:43161 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4487e3a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6cc2bc8bf733/172.17.0.3:0 2024-11-26T15:28:59,476 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:28:59,477 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:28:59,477 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:28:59,477 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=5, maxPoolSize=5 2024-11-26T15:28:59,477 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6cc2bc8bf733:0, corePoolSize=10, maxPoolSize=10 2024-11-26T15:28:59,477 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,477 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=2, maxPoolSize=2 2024-11-26T15:28:59,478 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,483 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732634969483 2024-11-26T15:28:59,485 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T15:28:59,487 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T15:28:59,490 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T15:28:59,491 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T15:28:59,492 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6cc2bc8bf733:43161 2024-11-26T15:28:59,492 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T15:28:59,492 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;6cc2bc8bf733:42689 2024-11-26T15:28:59,492 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T15:28:59,493 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6cc2bc8bf733:41249 2024-11-26T15:28:59,493 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T15:28:59,493 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T15:28:59,495 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T15:28:59,495 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T15:28:59,495 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T15:28:59,496 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T15:28:59,496 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T15:28:59,496 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T15:28:59,496 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T15:28:59,496 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T15:28:59,496 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T15:28:59,494 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,498 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:59,499 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(2659): reportForDuty to master=6cc2bc8bf733,44737,1732634937484 with port=43161, startcode=1732634938266 2024-11-26T15:28:59,499 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(2659): reportForDuty to master=6cc2bc8bf733,44737,1732634937484 with port=42689, startcode=1732634938394 2024-11-26T15:28:59,499 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(2659): reportForDuty to master=6cc2bc8bf733,44737,1732634937484 with port=41249, startcode=1732634938354 2024-11-26T15:28:59,499 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T15:28:59,505 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T15:28:59,506 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T15:28:59,507 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T15:28:59,511 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T15:28:59,512 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T15:28:59,512 DEBUG [RS:0;6cc2bc8bf733:43161 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T15:28:59,512 DEBUG [RS:1;6cc2bc8bf733:41249 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T15:28:59,512 DEBUG [RS:2;6cc2bc8bf733:42689 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T15:28:59,520 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.large.0-1732634939513,5,FailOnTimeoutGroup] 2024-11-26T15:28:59,521 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.small.0-1732634939521,5,FailOnTimeoutGroup] 2024-11-26T15:28:59,521 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,522 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T15:28:59,523 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,523 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741831_1007 (size=1321) 2024-11-26T15:28:59,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741831_1007 (size=1321) 2024-11-26T15:28:59,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741831_1007 (size=1321) 2024-11-26T15:28:59,562 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T15:28:59,562 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7 2024-11-26T15:28:59,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741832_1008 (size=32) 2024-11-26T15:28:59,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741832_1008 (size=32) 2024-11-26T15:28:59,580 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48677, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T15:28:59,580 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60931, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T15:28:59,580 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49229, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T15:28:59,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741832_1008 (size=32) 2024-11-26T15:28:59,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:28:59,587 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44737 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6cc2bc8bf733,42689,1732634938394 2024-11-26T15:28:59,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T15:28:59,590 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44737 {}] master.ServerManager(517): Registering regionserver=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:28:59,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T15:28:59,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:59,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:28:59,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T15:28:59,595 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T15:28:59,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:59,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:28:59,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T15:28:59,599 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T15:28:59,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:59,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:28:59,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T15:28:59,602 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44737 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6cc2bc8bf733,43161,1732634938266 2024-11-26T15:28:59,602 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44737 {}] master.ServerManager(517): Registering regionserver=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:28:59,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T15:28:59,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:28:59,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:28:59,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T15:28:59,606 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7 2024-11-26T15:28:59,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740 2024-11-26T15:28:59,607 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37937 2024-11-26T15:28:59,607 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T15:28:59,608 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44737 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6cc2bc8bf733,41249,1732634938354 2024-11-26T15:28:59,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740 2024-11-26T15:28:59,608 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44737 {}] master.ServerManager(517): Registering regionserver=6cc2bc8bf733,41249,1732634938354 2024-11-26T15:28:59,608 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7 2024-11-26T15:28:59,608 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37937 2024-11-26T15:28:59,608 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T15:28:59,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T15:28:59,613 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7 2024-11-26T15:28:59,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T15:28:59,613 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37937 2024-11-26T15:28:59,613 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T15:28:59,614 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T15:28:59,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T15:28:59,621 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:28:59,621 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72371817, jitterRate=0.0784241110086441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T15:28:59,625 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732634939584Initializing all the Stores at 1732634939586 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634939586Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634939587 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634939587Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634939587Cleaning up temporary data from old regions at 1732634939613 (+26 ms)Region opened successfully at 1732634939625 (+12 ms) 2024-11-26T15:28:59,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T15:28:59,626 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T15:28:59,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T15:28:59,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T15:28:59,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T15:28:59,627 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T15:28:59,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732634939626Disabling compacts and flushes for region at 1732634939626Disabling writes for close at 1732634939626Writing region close event to WAL at 1732634939627 (+1 ms)Closed at 1732634939627 2024-11-26T15:28:59,630 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T15:28:59,631 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T15:28:59,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T15:28:59,644 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T15:28:59,647 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T15:28:59,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T15:28:59,663 DEBUG [RS:0;6cc2bc8bf733:43161 {}] zookeeper.ZKUtil(111): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6cc2bc8bf733,43161,1732634938266 2024-11-26T15:28:59,663 DEBUG [RS:2;6cc2bc8bf733:42689 {}] zookeeper.ZKUtil(111): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6cc2bc8bf733,42689,1732634938394 2024-11-26T15:28:59,663 WARN [RS:0;6cc2bc8bf733:43161 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T15:28:59,663 WARN [RS:2;6cc2bc8bf733:42689 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T15:28:59,664 INFO [RS:0;6cc2bc8bf733:43161 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:28:59,664 INFO [RS:2;6cc2bc8bf733:42689 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:28:59,664 DEBUG [RS:1;6cc2bc8bf733:41249 {}] zookeeper.ZKUtil(111): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6cc2bc8bf733,41249,1732634938354 2024-11-26T15:28:59,664 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,42689,1732634938394 2024-11-26T15:28:59,664 WARN [RS:1;6cc2bc8bf733:41249 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T15:28:59,664 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266 2024-11-26T15:28:59,664 INFO [RS:1;6cc2bc8bf733:41249 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:28:59,664 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,41249,1732634938354 2024-11-26T15:28:59,665 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6cc2bc8bf733,43161,1732634938266] 2024-11-26T15:28:59,666 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6cc2bc8bf733,41249,1732634938354] 2024-11-26T15:28:59,666 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6cc2bc8bf733,42689,1732634938394] 2024-11-26T15:28:59,689 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T15:28:59,689 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T15:28:59,689 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T15:28:59,704 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T15:28:59,704 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T15:28:59,704 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T15:28:59,710 INFO [RS:1;6cc2bc8bf733:41249 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T15:28:59,710 INFO [RS:0;6cc2bc8bf733:43161 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T15:28:59,710 INFO [RS:2;6cc2bc8bf733:42689 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T15:28:59,710 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,710 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,710 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,711 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T15:28:59,711 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T15:28:59,712 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T15:28:59,717 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T15:28:59,717 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T15:28:59,717 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T15:28:59,718 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,718 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,718 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,718 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,718 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,718 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=2, maxPoolSize=2 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=2, maxPoolSize=2 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=2, maxPoolSize=2 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,719 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,720 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,720 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,720 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,720 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,720 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,720 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6cc2bc8bf733:0, corePoolSize=1, maxPoolSize=1 2024-11-26T15:28:59,720 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:28:59,720 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:28:59,720 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:28:59,720 DEBUG [RS:1;6cc2bc8bf733:41249 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:28:59,720 DEBUG [RS:0;6cc2bc8bf733:43161 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:28:59,720 DEBUG [RS:2;6cc2bc8bf733:42689 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6cc2bc8bf733:0, corePoolSize=3, maxPoolSize=3 2024-11-26T15:28:59,721 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,721 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,721 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,721 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,42689,1732634938394-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,722 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,43161,1732634938266-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T15:28:59,724 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,724 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,724 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,724 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,724 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,724 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,41249,1732634938354-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T15:28:59,741 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T15:28:59,741 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T15:28:59,742 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T15:28:59,743 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,41249,1732634938354-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,743 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,42689,1732634938394-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,743 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,43161,1732634938266-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,744 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,744 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,744 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,744 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.Replication(171): 6cc2bc8bf733,41249,1732634938354 started 2024-11-26T15:28:59,744 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.Replication(171): 6cc2bc8bf733,43161,1732634938266 started 2024-11-26T15:28:59,744 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.Replication(171): 6cc2bc8bf733,42689,1732634938394 started 2024-11-26T15:28:59,763 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,763 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1482): Serving as 6cc2bc8bf733,41249,1732634938354, RpcServer on 6cc2bc8bf733/172.17.0.3:41249, sessionid=0x10177c301390002 2024-11-26T15:28:59,764 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T15:28:59,764 DEBUG [RS:1;6cc2bc8bf733:41249 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6cc2bc8bf733,41249,1732634938354 2024-11-26T15:28:59,764 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,41249,1732634938354' 2024-11-26T15:28:59,764 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T15:28:59,765 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T15:28:59,766 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T15:28:59,766 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,766 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:28:59,766 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T15:28:59,766 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1482): Serving as 6cc2bc8bf733,42689,1732634938394, RpcServer on 6cc2bc8bf733/172.17.0.3:42689, sessionid=0x10177c301390003 2024-11-26T15:28:59,766 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1482): Serving as 6cc2bc8bf733,43161,1732634938266, RpcServer on 6cc2bc8bf733/172.17.0.3:43161, sessionid=0x10177c301390001 2024-11-26T15:28:59,766 DEBUG [RS:1;6cc2bc8bf733:41249 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6cc2bc8bf733,41249,1732634938354 2024-11-26T15:28:59,766 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,41249,1732634938354' 2024-11-26T15:28:59,766 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T15:28:59,766 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T15:28:59,766 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T15:28:59,766 DEBUG [RS:0;6cc2bc8bf733:43161 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6cc2bc8bf733,43161,1732634938266 2024-11-26T15:28:59,766 DEBUG [RS:2;6cc2bc8bf733:42689 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6cc2bc8bf733,42689,1732634938394 2024-11-26T15:28:59,766 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,43161,1732634938266' 2024-11-26T15:28:59,766 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,42689,1732634938394' 2024-11-26T15:28:59,766 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T15:28:59,766 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T15:28:59,767 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T15:28:59,767 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T15:28:59,767 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T15:28:59,767 DEBUG [RS:1;6cc2bc8bf733:41249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T15:28:59,767 INFO [RS:1;6cc2bc8bf733:41249 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T15:28:59,767 INFO [RS:1;6cc2bc8bf733:41249 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T15:28:59,768 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T15:28:59,768 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T15:28:59,768 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T15:28:59,768 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T15:28:59,768 DEBUG [RS:0;6cc2bc8bf733:43161 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6cc2bc8bf733,43161,1732634938266 2024-11-26T15:28:59,768 DEBUG [RS:2;6cc2bc8bf733:42689 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6cc2bc8bf733,42689,1732634938394 2024-11-26T15:28:59,768 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,43161,1732634938266' 2024-11-26T15:28:59,768 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6cc2bc8bf733,42689,1732634938394' 2024-11-26T15:28:59,768 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T15:28:59,768 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T15:28:59,768 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T15:28:59,768 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T15:28:59,769 DEBUG [RS:2;6cc2bc8bf733:42689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T15:28:59,769 DEBUG [RS:0;6cc2bc8bf733:43161 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T15:28:59,769 INFO [RS:0;6cc2bc8bf733:43161 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T15:28:59,769 INFO [RS:2;6cc2bc8bf733:42689 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T15:28:59,769 INFO [RS:0;6cc2bc8bf733:43161 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T15:28:59,769 INFO [RS:2;6cc2bc8bf733:42689 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T15:28:59,798 WARN [6cc2bc8bf733:44737 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T15:28:59,878 INFO [RS:1;6cc2bc8bf733:41249 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:28:59,878 INFO [RS:2;6cc2bc8bf733:42689 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:28:59,878 INFO [RS:0;6cc2bc8bf733:43161 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:28:59,881 INFO [RS:0;6cc2bc8bf733:43161 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C43161%2C1732634938266, suffix=, logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266, archiveDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs, maxLogs=32 2024-11-26T15:28:59,881 INFO [RS:2;6cc2bc8bf733:42689 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C42689%2C1732634938394, suffix=, logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,42689,1732634938394, archiveDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs, maxLogs=32 2024-11-26T15:28:59,881 INFO [RS:1;6cc2bc8bf733:41249 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C41249%2C1732634938354, suffix=, logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,41249,1732634938354, archiveDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs, maxLogs=32 2024-11-26T15:28:59,897 DEBUG [RS:0;6cc2bc8bf733:43161 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, exclude list is [], retry=0 2024-11-26T15:28:59,897 DEBUG [RS:1;6cc2bc8bf733:41249 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,41249,1732634938354/6cc2bc8bf733%2C41249%2C1732634938354.1732634939885, exclude list is [], retry=0 2024-11-26T15:28:59,898 DEBUG [RS:2;6cc2bc8bf733:42689 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,42689,1732634938394/6cc2bc8bf733%2C42689%2C1732634938394.1732634939885, exclude list is [], retry=0 2024-11-26T15:28:59,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:28:59,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:28:59,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:28:59,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:28:59,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:28:59,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:28:59,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:28:59,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:28:59,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:28:59,912 INFO [RS:2;6cc2bc8bf733:42689 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,42689,1732634938394/6cc2bc8bf733%2C42689%2C1732634938394.1732634939885 2024-11-26T15:28:59,912 INFO [RS:0;6cc2bc8bf733:43161 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 2024-11-26T15:28:59,912 INFO [RS:1;6cc2bc8bf733:41249 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,41249,1732634938354/6cc2bc8bf733%2C41249%2C1732634938354.1732634939885 2024-11-26T15:28:59,913 DEBUG [RS:1;6cc2bc8bf733:41249 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:28:59,913 DEBUG [RS:0;6cc2bc8bf733:43161 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:28:59,914 DEBUG [RS:2;6cc2bc8bf733:42689 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:00,051 DEBUG [6cc2bc8bf733:44737 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-26T15:29:00,061 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(204): Hosts are {6cc2bc8bf733=0} racks are {/default-rack=0} 2024-11-26T15:29:00,068 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T15:29:00,068 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T15:29:00,068 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T15:29:00,068 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T15:29:00,068 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T15:29:00,068 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T15:29:00,068 INFO [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T15:29:00,068 INFO [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T15:29:00,068 INFO [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T15:29:00,068 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T15:29:00,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:00,080 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6cc2bc8bf733,43161,1732634938266, state=OPENING 2024-11-26T15:29:00,137 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T15:29:00,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:00,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:00,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:00,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:00,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,149 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,150 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T15:29:00,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,43161,1732634938266}] 2024-11-26T15:29:00,329 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T15:29:00,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59811, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T15:29:00,344 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T15:29:00,344 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:00,345 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T15:29:00,348 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C43161%2C1732634938266.meta, suffix=.meta, logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266, archiveDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs, maxLogs=32 2024-11-26T15:29:00,363 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, exclude list is [], retry=0 2024-11-26T15:29:00,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:00,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:00,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:00,370 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta 2024-11-26T15:29:00,371 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:00,371 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:00,373 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T15:29:00,375 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T15:29:00,379 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T15:29:00,383 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T15:29:00,383 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:00,383 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T15:29:00,384 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T15:29:00,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T15:29:00,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T15:29:00,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:00,389 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:00,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T15:29:00,391 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T15:29:00,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:00,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:00,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T15:29:00,393 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T15:29:00,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:00,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:00,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T15:29:00,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T15:29:00,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:00,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:00,396 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T15:29:00,398 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740 2024-11-26T15:29:00,401 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740 2024-11-26T15:29:00,404 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T15:29:00,404 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T15:29:00,405 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T15:29:00,407 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T15:29:00,409 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64669021, jitterRate=-0.03635649383068085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T15:29:00,409 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T15:29:00,411 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732634940384Writing region info on filesystem at 1732634940384Initializing all the Stores at 1732634940386 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634940386Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634940387 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634940387Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634940387Cleaning up temporary data from old regions at 1732634940404 (+17 ms)Running coprocessor post-open hooks at 1732634940409 (+5 ms)Region opened successfully at 1732634940411 (+2 ms) 2024-11-26T15:29:00,418 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732634940321 2024-11-26T15:29:00,428 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T15:29:00,428 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T15:29:00,430 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:00,433 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6cc2bc8bf733,43161,1732634938266, state=OPEN 2024-11-26T15:29:00,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:00,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:00,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:00,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:00,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:00,443 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:00,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T15:29:00,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,43161,1732634938266 in 291 msec 2024-11-26T15:29:00,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T15:29:00,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 813 msec 2024-11-26T15:29:00,458 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T15:29:00,458 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T15:29:00,476 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:29:00,477 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1] 2024-11-26T15:29:00,495 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:29:00,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44457, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:29:00,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1250 sec 2024-11-26T15:29:00,515 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732634940515, completionTime=-1 2024-11-26T15:29:00,519 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-26T15:29:00,520 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T15:29:00,549 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-26T15:29:00,549 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732635000549 2024-11-26T15:29:00,549 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732635060549 2024-11-26T15:29:00,549 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-26T15:29:00,551 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T15:29:00,575 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,44737,1732634937484-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:00,576 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,44737,1732634937484-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:00,576 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,44737,1732634937484-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:00,578 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6cc2bc8bf733:44737, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:00,578 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:00,579 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:00,587 DEBUG [master/6cc2bc8bf733:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T15:29:00,618 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.111sec 2024-11-26T15:29:00,619 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T15:29:00,620 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T15:29:00,621 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T15:29:00,621 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T15:29:00,621 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T15:29:00,622 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,44737,1732634937484-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T15:29:00,623 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,44737,1732634937484-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T15:29:00,627 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T15:29:00,628 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T15:29:00,629 INFO [master/6cc2bc8bf733:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6cc2bc8bf733,44737,1732634937484-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:00,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68674fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:29:00,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6cc2bc8bf733,44737,-1 for getting cluster id 2024-11-26T15:29:00,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T15:29:00,662 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b86a04a6-48ba-44c0-ade9-bd78fee725c1' 2024-11-26T15:29:00,664 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T15:29:00,665 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b86a04a6-48ba-44c0-ade9-bd78fee725c1" 2024-11-26T15:29:00,667 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b32dc23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:29:00,667 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6cc2bc8bf733,44737,-1] 2024-11-26T15:29:00,669 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T15:29:00,671 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:00,672 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T15:29:00,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb9e3a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T15:29:00,676 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:29:00,685 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1] 2024-11-26T15:29:00,685 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:29:00,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:29:00,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6cc2bc8bf733,44737,1732634937484 2024-11-26T15:29:00,705 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:37937/hbase 2024-11-26T15:29:00,721 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=9809 2024-11-26T15:29:00,739 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:00,743 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:00,744 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:00,749 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54430776, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-54430776, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:00,771 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54430776/hregion-54430776.1732634940751, exclude list is [], retry=0 2024-11-26T15:29:00,775 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:00,775 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:00,776 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:00,783 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54430776/hregion-54430776.1732634940751 2024-11-26T15:29:00,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:00,785 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3942bc2cc360772c4d3208cc44c61165, NAME => 'testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:00,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741838_1014 (size=64) 2024-11-26T15:29:00,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741838_1014 (size=64) 2024-11-26T15:29:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741838_1014 (size=64) 2024-11-26T15:29:00,802 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:00,804 INFO [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:00,807 INFO [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3942bc2cc360772c4d3208cc44c61165 columnFamilyName a 2024-11-26T15:29:00,808 DEBUG [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:00,809 INFO [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] regionserver.HStore(327): Store=3942bc2cc360772c4d3208cc44c61165/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:00,809 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:00,811 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:00,812 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:00,813 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:00,813 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:00,817 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:00,821 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:00,822 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3942bc2cc360772c4d3208cc44c61165; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72577823, jitterRate=0.08149383962154388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:00,822 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3942bc2cc360772c4d3208cc44c61165: Writing region info on filesystem at 1732634940802Initializing all the Stores at 1732634940804 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634940804Cleaning up temporary data from old regions at 1732634940813 (+9 ms)Region opened successfully at 1732634940822 (+9 ms) 2024-11-26T15:29:00,823 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3942bc2cc360772c4d3208cc44c61165, disabling compactions & flushes 2024-11-26T15:29:00,823 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165. 2024-11-26T15:29:00,823 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165. 2024-11-26T15:29:00,823 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165. after waiting 0 ms 2024-11-26T15:29:00,823 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165. 2024-11-26T15:29:00,823 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165. 2024-11-26T15:29:00,823 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3942bc2cc360772c4d3208cc44c61165: Waiting for close lock at 1732634940822Disabling compacts and flushes for region at 1732634940822Disabling writes for close at 1732634940823 (+1 ms)Writing region close event to WAL at 1732634940823Closed at 1732634940823 2024-11-26T15:29:00,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741837_1013 (size=93) 2024-11-26T15:29:00,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741837_1013 (size=93) 2024-11-26T15:29:00,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741837_1013 (size=93) 2024-11-26T15:29:00,838 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:00,838 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54430776:(num 1732634940751) 2024-11-26T15:29:00,841 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T15:29:00,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741839_1015 (size=369) 2024-11-26T15:29:00,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741839_1015 (size=369) 2024-11-26T15:29:00,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741839_1015 (size=369) 2024-11-26T15:29:00,859 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T15:29:00,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741840_1016 (size=231) 2024-11-26T15:29:00,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741840_1016 (size=231) 2024-11-26T15:29:00,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741840_1016 (size=231) 2024-11-26T15:29:00,896 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1, size=369 (369bytes) 2024-11-26T15:29:00,896 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T15:29:00,896 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T15:29:00,897 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1 2024-11-26T15:29:00,902 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1 after 3ms 2024-11-26T15:29:00,908 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1 took 14ms 2024-11-26T15:29:00,912 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1 so closing down 2024-11-26T15:29:00,916 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T15:29:00,917 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T15:29:00,918 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:00,918 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:00,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741841_1017 (size=369) 2024-11-26T15:29:00,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741841_1017 (size=369) 2024-11-26T15:29:00,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741841_1017 (size=369) 2024-11-26T15:29:00,930 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:00,933 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002 2024-11-26T15:29:00,938 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-26T15:29:00,938 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1, journal: Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1, size=369 (369bytes) at 1732634940896Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1 so closing down at 1732634940912 (+16 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000001-wal-1.temp at 1732634940917 (+5 ms)3 split writer threads finished at 1732634940918 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732634940931 (+13 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002 at 1732634940933 (+2 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732634940938 (+5 ms) 2024-11-26T15:29:00,953 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2, size=231 (231bytes) 2024-11-26T15:29:00,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2 2024-11-26T15:29:00,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2 after 0ms 2024-11-26T15:29:00,958 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2 took 5ms 2024-11-26T15:29:00,960 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2 so closing down 2024-11-26T15:29:00,960 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:00,963 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T15:29:00,965 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T15:29:00,965 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:00,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741842_1018 (size=231) 2024-11-26T15:29:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741842_1018 (size=231) 2024-11-26T15:29:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741842_1018 (size=231) 2024-11-26T15:29:00,974 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:00,981 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-26T15:29:00,983 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-26T15:29:00,983 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2, journal: Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2, size=231 (231bytes) at 1732634940953Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2 so closing down at 1732634940960 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002-wal-2.temp at 1732634940965 (+5 ms)3 split writer threads finished at 1732634940965Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732634940974 (+9 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732634940983 (+9 ms) 2024-11-26T15:29:00,983 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:00,986 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:01,002 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal.1732634940987, exclude list is [], retry=0 2024-11-26T15:29:01,006 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:01,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:01,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:01,009 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal.1732634940987 2024-11-26T15:29:01,010 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:01,010 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3942bc2cc360772c4d3208cc44c61165, NAME => 'testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:01,010 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:01,010 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,010 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,012 INFO [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,014 INFO [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3942bc2cc360772c4d3208cc44c61165 columnFamilyName a 2024-11-26T15:29:01,014 DEBUG [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:01,015 INFO [StoreOpener-3942bc2cc360772c4d3208cc44c61165-1 {}] regionserver.HStore(327): Store=3942bc2cc360772c4d3208cc44c61165/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:01,015 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,017 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,022 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002 2024-11-26T15:29:01,030 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002 2024-11-26T15:29:01,033 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3942bc2cc360772c4d3208cc44c61165 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T15:29:01,087 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/.tmp/a/9eb14abb23bb4421973978425f9b97c6 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732634940838/Put/seqid=0 2024-11-26T15:29:01,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741844_1020 (size=5170) 2024-11-26T15:29:01,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741844_1020 (size=5170) 2024-11-26T15:29:01,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741844_1020 (size=5170) 2024-11-26T15:29:01,104 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/.tmp/a/9eb14abb23bb4421973978425f9b97c6 2024-11-26T15:29:01,152 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/.tmp/a/9eb14abb23bb4421973978425f9b97c6 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/a/9eb14abb23bb4421973978425f9b97c6 2024-11-26T15:29:01,162 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/a/9eb14abb23bb4421973978425f9b97c6, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T15:29:01,169 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3942bc2cc360772c4d3208cc44c61165 in 133ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T15:29:01,171 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/0000000000000000002 2024-11-26T15:29:01,172 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,172 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,176 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3942bc2cc360772c4d3208cc44c61165 2024-11-26T15:29:01,180 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/3942bc2cc360772c4d3208cc44c61165/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T15:29:01,182 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3942bc2cc360772c4d3208cc44c61165; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71698312, jitterRate=0.06838810443878174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:01,183 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3942bc2cc360772c4d3208cc44c61165: Writing region info on filesystem at 1732634941010Initializing all the Stores at 1732634941012 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634941012Obtaining lock to block concurrent updates at 1732634941033 (+21 ms)Preparing flush snapshotting stores in 3942bc2cc360772c4d3208cc44c61165 at 1732634941033Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732634941036 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732634940740.3942bc2cc360772c4d3208cc44c61165. at 1732634941036Flushing 3942bc2cc360772c4d3208cc44c61165/a: creating writer at 1732634941038 (+2 ms)Flushing 3942bc2cc360772c4d3208cc44c61165/a: appending metadata at 1732634941072 (+34 ms)Flushing 3942bc2cc360772c4d3208cc44c61165/a: closing flushed file at 1732634941076 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65e2dbe5: reopening flushed file at 1732634941150 (+74 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3942bc2cc360772c4d3208cc44c61165 in 133ms, sequenceid=2, compaction requested=false; wal=null at 1732634941169 (+19 ms)Cleaning up temporary data from old regions at 1732634941172 (+3 ms)Region opened successfully at 1732634941183 (+11 ms) 2024-11-26T15:29:01,215 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 363) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:37937/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:40124 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:48014 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:40282 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:58602 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47858 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:58692 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=9791 (was 9809) 2024-11-26T15:29:01,226 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=9790 2024-11-26T15:29:01,241 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:01,243 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:01,244 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:01,248 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-56055416, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-56055416, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:01,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-56055416/hregion-56055416.1732634941249, exclude list is [], retry=0 2024-11-26T15:29:01,264 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:01,264 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:01,265 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:01,267 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-56055416/hregion-56055416.1732634941249 2024-11-26T15:29:01,268 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:01,268 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1bd477782b884e56170624153c60c63c, NAME => 'testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:01,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741846_1022 (size=64) 2024-11-26T15:29:01,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741846_1022 (size=64) 2024-11-26T15:29:01,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741846_1022 (size=64) 2024-11-26T15:29:01,283 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:01,285 INFO [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,288 INFO [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bd477782b884e56170624153c60c63c columnFamilyName a 2024-11-26T15:29:01,288 DEBUG [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:01,289 INFO [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] regionserver.HStore(327): Store=1bd477782b884e56170624153c60c63c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:01,289 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,291 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,291 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,292 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,292 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,294 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,297 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:01,298 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1bd477782b884e56170624153c60c63c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71687944, jitterRate=0.06823360919952393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:01,298 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1bd477782b884e56170624153c60c63c: Writing region info on filesystem at 1732634941283Initializing all the Stores at 1732634941285 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634941285Cleaning up temporary data from old regions at 1732634941292 (+7 ms)Region opened successfully at 1732634941298 (+6 ms) 2024-11-26T15:29:01,299 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1bd477782b884e56170624153c60c63c, disabling compactions & flushes 2024-11-26T15:29:01,299 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c. 2024-11-26T15:29:01,299 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c. 2024-11-26T15:29:01,299 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c. after waiting 0 ms 2024-11-26T15:29:01,299 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c. 2024-11-26T15:29:01,299 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c. 2024-11-26T15:29:01,299 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1bd477782b884e56170624153c60c63c: Waiting for close lock at 1732634941299Disabling compacts and flushes for region at 1732634941299Disabling writes for close at 1732634941299Writing region close event to WAL at 1732634941299Closed at 1732634941299 2024-11-26T15:29:01,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741845_1021 (size=93) 2024-11-26T15:29:01,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741845_1021 (size=93) 2024-11-26T15:29:01,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741845_1021 (size=93) 2024-11-26T15:29:01,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:01,306 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-56055416:(num 1732634941249) 2024-11-26T15:29:01,307 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T15:29:01,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741847_1023 (size=369) 2024-11-26T15:29:01,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741847_1023 (size=369) 2024-11-26T15:29:01,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741847_1023 (size=369) 2024-11-26T15:29:01,321 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T15:29:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741848_1024 (size=231) 2024-11-26T15:29:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741848_1024 (size=231) 2024-11-26T15:29:01,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741848_1024 (size=231) 2024-11-26T15:29:01,350 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2, size=231 (231bytes) 2024-11-26T15:29:01,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2 2024-11-26T15:29:01,351 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2 after 1ms 2024-11-26T15:29:01,354 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2 took 4ms 2024-11-26T15:29:01,357 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2 so closing down 2024-11-26T15:29:01,357 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:01,359 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T15:29:01,361 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T15:29:01,361 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:01,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741849_1025 (size=231) 2024-11-26T15:29:01,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741849_1025 (size=231) 2024-11-26T15:29:01,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741849_1025 (size=231) 2024-11-26T15:29:01,370 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:01,373 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002 2024-11-26T15:29:01,373 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-26T15:29:01,373 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2, journal: Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2, size=231 (231bytes) at 1732634941350Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2 so closing down at 1732634941357 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002-wal-2.temp at 1732634941361 (+4 ms)3 split writer threads finished at 1732634941361Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732634941370 (+9 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002 at 1732634941373 (+3 ms)Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732634941373 2024-11-26T15:29:01,389 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1, size=369 (369bytes) 2024-11-26T15:29:01,389 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1 2024-11-26T15:29:01,390 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1 after 1ms 2024-11-26T15:29:01,393 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1 took 5ms 2024-11-26T15:29:01,396 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1 so closing down 2024-11-26T15:29:01,396 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:01,399 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T15:29:01,400 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T15:29:01,401 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:01,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741850_1026 (size=369) 2024-11-26T15:29:01,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741850_1026 (size=369) 2024-11-26T15:29:01,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741850_1026 (size=369) 2024-11-26T15:29:01,409 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:01,415 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002, length=231 2024-11-26T15:29:01,418 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002 2024-11-26T15:29:01,418 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-26T15:29:01,418 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1, journal: Splitting hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1, size=369 (369bytes) at 1732634941389Finishing writing output for hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1 so closing down at 1732634941396 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000001-wal-1.temp at 1732634941400 (+4 ms)3 split writer threads finished at 1732634941401 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732634941409 (+8 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002 at 1732634941418 (+9 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732634941418 2024-11-26T15:29:01,419 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:01,421 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:01,435 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal.1732634941422, exclude list is [], retry=0 2024-11-26T15:29:01,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:01,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:01,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:01,442 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal.1732634941422 2024-11-26T15:29:01,443 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:01,443 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bd477782b884e56170624153c60c63c, NAME => 'testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:01,443 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:01,443 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,443 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,445 INFO [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,447 INFO [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bd477782b884e56170624153c60c63c columnFamilyName a 2024-11-26T15:29:01,448 DEBUG [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:01,448 INFO [StoreOpener-1bd477782b884e56170624153c60c63c-1 {}] regionserver.HStore(327): Store=1bd477782b884e56170624153c60c63c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:01,449 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,450 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,452 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,453 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002 2024-11-26T15:29:01,457 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002 2024-11-26T15:29:01,457 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1bd477782b884e56170624153c60c63c 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T15:29:01,472 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/.tmp/a/82a45c80371d47ceb42298254a6604a6 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732634941306/Put/seqid=0 2024-11-26T15:29:01,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741852_1028 (size=5170) 2024-11-26T15:29:01,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741852_1028 (size=5170) 2024-11-26T15:29:01,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741852_1028 (size=5170) 2024-11-26T15:29:01,483 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/.tmp/a/82a45c80371d47ceb42298254a6604a6 2024-11-26T15:29:01,491 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/.tmp/a/82a45c80371d47ceb42298254a6604a6 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/a/82a45c80371d47ceb42298254a6604a6 2024-11-26T15:29:01,499 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/a/82a45c80371d47ceb42298254a6604a6, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T15:29:01,499 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1bd477782b884e56170624153c60c63c in 42ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T15:29:01,500 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/0000000000000000002 2024-11-26T15:29:01,501 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,501 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,503 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1bd477782b884e56170624153c60c63c 2024-11-26T15:29:01,507 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/1bd477782b884e56170624153c60c63c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T15:29:01,508 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1bd477782b884e56170624153c60c63c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67053125, jitterRate=-8.305758237838745E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:01,509 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1bd477782b884e56170624153c60c63c: Writing region info on filesystem at 1732634941443Initializing all the Stores at 1732634941445 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634941445Obtaining lock to block concurrent updates at 1732634941457 (+12 ms)Preparing flush snapshotting stores in 1bd477782b884e56170624153c60c63c at 1732634941457Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732634941457Flushing stores of testReplayEditsWrittenIntoWAL,,1732634941242.1bd477782b884e56170624153c60c63c. at 1732634941457Flushing 1bd477782b884e56170624153c60c63c/a: creating writer at 1732634941458 (+1 ms)Flushing 1bd477782b884e56170624153c60c63c/a: appending metadata at 1732634941471 (+13 ms)Flushing 1bd477782b884e56170624153c60c63c/a: closing flushed file at 1732634941471Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7488c414: reopening flushed file at 1732634941490 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1bd477782b884e56170624153c60c63c in 42ms, sequenceid=2, compaction requested=false; wal=null at 1732634941499 (+9 ms)Cleaning up temporary data from old regions at 1732634941501 (+2 ms)Region opened successfully at 1732634941509 (+8 ms) 2024-11-26T15:29:01,525 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:40364 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:58782 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:40124 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:58602 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:48108 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47858 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=9785 (was 9790) 2024-11-26T15:29:01,535 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=9784 2024-11-26T15:29:01,550 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:01,553 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:01,553 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:01,557 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15772969, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-15772969, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:01,569 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15772969/hregion-15772969.1732634941557, exclude list is [], retry=0 2024-11-26T15:29:01,573 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:01,573 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:01,573 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:01,576 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15772969/hregion-15772969.1732634941557 2024-11-26T15:29:01,576 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:01,576 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => eeec13b53dcb44d0343f5d7201fb29b5, NAME => 'testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741854_1030 (size=64) 2024-11-26T15:29:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741854_1030 (size=64) 2024-11-26T15:29:01,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741854_1030 (size=64) 2024-11-26T15:29:01,588 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:01,590 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,592 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeec13b53dcb44d0343f5d7201fb29b5 columnFamilyName a 2024-11-26T15:29:01,592 DEBUG [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:01,592 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(327): Store=eeec13b53dcb44d0343f5d7201fb29b5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:01,592 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,594 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeec13b53dcb44d0343f5d7201fb29b5 columnFamilyName b 2024-11-26T15:29:01,594 DEBUG [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:01,595 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(327): Store=eeec13b53dcb44d0343f5d7201fb29b5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:01,595 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,597 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeec13b53dcb44d0343f5d7201fb29b5 columnFamilyName c 2024-11-26T15:29:01,597 DEBUG [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:01,598 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(327): Store=eeec13b53dcb44d0343f5d7201fb29b5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:01,598 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,599 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,599 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,600 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,600 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,601 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:01,603 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:01,605 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:01,606 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eeec13b53dcb44d0343f5d7201fb29b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70645252, jitterRate=0.052696287631988525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:01,607 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eeec13b53dcb44d0343f5d7201fb29b5: Writing region info on filesystem at 1732634941588Initializing all the Stores at 1732634941589 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634941589Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634941590 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634941590Cleaning up temporary data from old regions at 1732634941600 (+10 ms)Region opened successfully at 1732634941606 (+6 ms) 2024-11-26T15:29:01,607 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eeec13b53dcb44d0343f5d7201fb29b5, disabling compactions & flushes 2024-11-26T15:29:01,607 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:01,607 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:01,607 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. after waiting 0 ms 2024-11-26T15:29:01,607 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:01,608 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:01,608 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eeec13b53dcb44d0343f5d7201fb29b5: Waiting for close lock at 1732634941607Disabling compacts and flushes for region at 1732634941607Disabling writes for close at 1732634941607Writing region close event to WAL at 1732634941608 (+1 ms)Closed at 1732634941608 2024-11-26T15:29:01,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741853_1029 (size=93) 2024-11-26T15:29:01,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741853_1029 (size=93) 2024-11-26T15:29:01,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741853_1029 (size=93) 2024-11-26T15:29:01,615 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:01,615 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15772969:(num 1732634941557) 2024-11-26T15:29:01,615 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:01,618 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:01,632 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619, exclude list is [], retry=0 2024-11-26T15:29:01,635 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:01,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:01,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:01,639 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 2024-11-26T15:29:01,639 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:01,872 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619, size=0 (0bytes) 2024-11-26T15:29:01,872 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 might be still open, length is 0 2024-11-26T15:29:01,872 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 2024-11-26T15:29:01,873 WARN [IPC Server handler 2 on default port 37937 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-26T15:29:01,874 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 after 1ms 2024-11-26T15:29:02,635 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:48146 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39747:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48146 dst: /127.0.0.1:39747 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39747 remote=/127.0.0.1:48146]. Total timeout mills is 60000, 59199 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:02,636 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:40392 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34329:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40392 dst: /127.0.0.1:34329 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:02,636 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:58838 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35535:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58838 dst: /127.0.0.1:35535 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:02,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741855_1032 (size=470675) 2024-11-26T15:29:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741855_1032 (size=470675) 2024-11-26T15:29:02,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741855_1032 (size=470675) 2024-11-26T15:29:05,875 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 after 4002ms 2024-11-26T15:29:05,881 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 took 4010ms 2024-11-26T15:29:05,887 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732634941619.temp 2024-11-26T15:29:05,892 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000000001-wal.1732634941619.temp 2024-11-26T15:29:05,901 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:29:05,961 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619; continuing. 2024-11-26T15:29:05,962 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 so closing down 2024-11-26T15:29:05,962 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:05,962 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:05,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741856_1033 (size=470683) 2024-11-26T15:29:05,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741856_1033 (size=470683) 2024-11-26T15:29:05,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741856_1033 (size=470683) 2024-11-26T15:29:05,968 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000000001-wal.1732634941619.temp (wrote 3002 edits, skipped 0 edits in 30 ms) 2024-11-26T15:29:05,970 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000000001-wal.1732634941619.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000003002 2024-11-26T15:29:05,970 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T15:29:05,970 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619, journal: Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619, size=0 (0bytes) at 1732634941872Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000000001-wal.1732634941619.temp at 1732634945892 (+4020 ms)Split 1024 edits, skipped 0 edits. at 1732634945936 (+44 ms)Split 2048 edits, skipped 0 edits. at 1732634945949 (+13 ms)Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 so closing down at 1732634945962 (+13 ms)3 split writer threads finished at 1732634945962Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000000001-wal.1732634941619.temp (wrote 3002 edits, skipped 0 edits in 30 ms) at 1732634945968 (+6 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000000001-wal.1732634941619.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000003002 at 1732634945970 (+2 ms)Processed 3002 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619, size=0, length=0, corrupted=false, cancelled=false at 1732634945970 2024-11-26T15:29:05,973 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634941619 2024-11-26T15:29:05,974 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000003002 2024-11-26T15:29:05,975 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:05,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:05,989 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T15:29:05,995 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634945978, exclude list is [], retry=0 2024-11-26T15:29:05,998 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:05,999 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:06,000 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:06,010 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634945978 2024-11-26T15:29:06,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:06,011 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:06,013 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,014 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeec13b53dcb44d0343f5d7201fb29b5 columnFamilyName a 2024-11-26T15:29:06,014 DEBUG [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:06,014 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(327): Store=eeec13b53dcb44d0343f5d7201fb29b5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:06,015 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,015 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeec13b53dcb44d0343f5d7201fb29b5 columnFamilyName b 2024-11-26T15:29:06,016 DEBUG [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:06,016 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(327): Store=eeec13b53dcb44d0343f5d7201fb29b5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:06,016 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,017 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeec13b53dcb44d0343f5d7201fb29b5 columnFamilyName c 2024-11-26T15:29:06,017 DEBUG [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:06,018 INFO [StoreOpener-eeec13b53dcb44d0343f5d7201fb29b5-1 {}] regionserver.HStore(327): Store=eeec13b53dcb44d0343f5d7201fb29b5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:06,018 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,022 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000003002 2024-11-26T15:29:06,066 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T15:29:06,408 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eeec13b53dcb44d0343f5d7201fb29b5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T15:29:06,446 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/a/0113f7f9646549d08825c72290ea3485 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732634941650/Put/seqid=0 2024-11-26T15:29:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741858_1035 (size=50463) 2024-11-26T15:29:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741858_1035 (size=50463) 2024-11-26T15:29:06,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741858_1035 (size=50463) 2024-11-26T15:29:06,454 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/a/0113f7f9646549d08825c72290ea3485 2024-11-26T15:29:06,462 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/a/0113f7f9646549d08825c72290ea3485 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/a/0113f7f9646549d08825c72290ea3485 2024-11-26T15:29:06,470 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/a/0113f7f9646549d08825c72290ea3485, entries=754, sequenceid=754, filesize=49.3 K 2024-11-26T15:29:06,470 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for eeec13b53dcb44d0343f5d7201fb29b5 in 63ms, sequenceid=754, compaction requested=false; wal=null 2024-11-26T15:29:06,485 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T15:29:06,485 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eeec13b53dcb44d0343f5d7201fb29b5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T15:29:06,494 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/a/0c713a1379784d4d95685f9ca792249c is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732634941708/Put/seqid=0 2024-11-26T15:29:06,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741859_1036 (size=20072) 2024-11-26T15:29:06,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741859_1036 (size=20072) 2024-11-26T15:29:06,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741859_1036 (size=20072) 2024-11-26T15:29:06,505 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/a/0c713a1379784d4d95685f9ca792249c 2024-11-26T15:29:06,530 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/b/e820e99254c04c37b84ea4fe8f8eb8a2 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732634941730/Put/seqid=0 2024-11-26T15:29:06,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741860_1037 (size=35835) 2024-11-26T15:29:06,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741860_1037 (size=35835) 2024-11-26T15:29:06,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741860_1037 (size=35835) 2024-11-26T15:29:06,538 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/b/e820e99254c04c37b84ea4fe8f8eb8a2 2024-11-26T15:29:06,548 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/a/0c713a1379784d4d95685f9ca792249c as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/a/0c713a1379784d4d95685f9ca792249c 2024-11-26T15:29:06,555 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/a/0c713a1379784d4d95685f9ca792249c, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-26T15:29:06,556 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/b/e820e99254c04c37b84ea4fe8f8eb8a2 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/b/e820e99254c04c37b84ea4fe8f8eb8a2 2024-11-26T15:29:06,563 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/b/e820e99254c04c37b84ea4fe8f8eb8a2, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-26T15:29:06,564 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for eeec13b53dcb44d0343f5d7201fb29b5 in 78ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-26T15:29:06,580 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T15:29:06,580 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eeec13b53dcb44d0343f5d7201fb29b5 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T15:29:06,590 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/b/616c80a7eb704fd892deef8ed528e485 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732634941752/Put/seqid=0 2024-11-26T15:29:06,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741861_1038 (size=35082) 2024-11-26T15:29:06,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741861_1038 (size=35082) 2024-11-26T15:29:06,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741861_1038 (size=35082) 2024-11-26T15:29:06,600 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/b/616c80a7eb704fd892deef8ed528e485 2024-11-26T15:29:06,623 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/c/03d06f53492d4a6cbcb561defb7be7d1 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732634941787/Put/seqid=0 2024-11-26T15:29:06,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741862_1039 (size=20825) 2024-11-26T15:29:06,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741862_1039 (size=20825) 2024-11-26T15:29:06,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741862_1039 (size=20825) 2024-11-26T15:29:06,632 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/c/03d06f53492d4a6cbcb561defb7be7d1 2024-11-26T15:29:06,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/b/616c80a7eb704fd892deef8ed528e485 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/b/616c80a7eb704fd892deef8ed528e485 2024-11-26T15:29:06,647 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/b/616c80a7eb704fd892deef8ed528e485, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-26T15:29:06,649 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/c/03d06f53492d4a6cbcb561defb7be7d1 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/c/03d06f53492d4a6cbcb561defb7be7d1 2024-11-26T15:29:06,656 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/c/03d06f53492d4a6cbcb561defb7be7d1, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-26T15:29:06,656 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for eeec13b53dcb44d0343f5d7201fb29b5 in 76ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-26T15:29:06,667 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732634941832/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:06,671 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000003002 2024-11-26T15:29:06,672 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T15:29:06,672 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eeec13b53dcb44d0343f5d7201fb29b5 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-26T15:29:06,682 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/c/4eef298ff1f744ab8f585169b0409a7e is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732634941795/Put/seqid=0 2024-11-26T15:29:06,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741863_1040 (size=50301) 2024-11-26T15:29:06,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741863_1040 (size=50301) 2024-11-26T15:29:06,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741863_1040 (size=50301) 2024-11-26T15:29:06,691 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/c/4eef298ff1f744ab8f585169b0409a7e 2024-11-26T15:29:06,698 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4eef298ff1f744ab8f585169b0409a7e 2024-11-26T15:29:06,699 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/.tmp/c/4eef298ff1f744ab8f585169b0409a7e as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/c/4eef298ff1f744ab8f585169b0409a7e 2024-11-26T15:29:06,706 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4eef298ff1f744ab8f585169b0409a7e 2024-11-26T15:29:06,707 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/c/4eef298ff1f744ab8f585169b0409a7e, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-26T15:29:06,707 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for eeec13b53dcb44d0343f5d7201fb29b5 in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-26T15:29:06,708 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/0000000000000003002 2024-11-26T15:29:06,710 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,710 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,711 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T15:29:06,713 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eeec13b53dcb44d0343f5d7201fb29b5 2024-11-26T15:29:06,715 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenIntoWAL/eeec13b53dcb44d0343f5d7201fb29b5/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-26T15:29:06,716 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eeec13b53dcb44d0343f5d7201fb29b5; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61120210, jitterRate=-0.08923789858818054}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T15:29:06,717 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eeec13b53dcb44d0343f5d7201fb29b5: Writing region info on filesystem at 1732634946011Initializing all the Stores at 1732634946012 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634946012Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634946013 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634946013Cleaning up temporary data from old regions at 1732634946710 (+697 ms)Region opened successfully at 1732634946716 (+6 ms) 2024-11-26T15:29:06,788 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eeec13b53dcb44d0343f5d7201fb29b5, disabling compactions & flushes 2024-11-26T15:29:06,788 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:06,788 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:06,788 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. after waiting 0 ms 2024-11-26T15:29:06,788 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:06,791 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732634941551.eeec13b53dcb44d0343f5d7201fb29b5. 2024-11-26T15:29:06,791 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eeec13b53dcb44d0343f5d7201fb29b5: Waiting for close lock at 1732634946788Disabling compacts and flushes for region at 1732634946788Disabling writes for close at 1732634946788Writing region close event to WAL at 1732634946790 (+2 ms)Closed at 1732634946790 2024-11-26T15:29:06,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741857_1034 (size=93) 2024-11-26T15:29:06,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741857_1034 (size=93) 2024-11-26T15:29:06,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741857_1034 (size=93) 2024-11-26T15:29:06,798 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:06,798 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732634945978) 2024-11-26T15:29:06,814 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=398 (was 384) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:37445 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:44377 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-855092889_22 at /127.0.0.1:59620 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-855092889_22 at /127.0.0.1:33368 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-855092889_22 at /127.0.0.1:59612 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44377 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:37937 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:37937 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37445 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-855092889_22 at /127.0.0.1:47728 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=9722 (was 9784) 2024-11-26T15:29:06,827 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=398, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=9721 2024-11-26T15:29:06,843 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:06,846 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:06,847 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:06,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58018778, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-58018778, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:06,862 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58018778/hregion-58018778.1732634946850, exclude list is [], retry=0 2024-11-26T15:29:06,865 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:06,866 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:06,866 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:06,869 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58018778/hregion-58018778.1732634946850 2024-11-26T15:29:06,869 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:06,869 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e3bc0d04ec09592c2469db6cdfe741ec, NAME => 'test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:06,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741865_1042 (size=43) 2024-11-26T15:29:06,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741865_1042 (size=43) 2024-11-26T15:29:06,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741865_1042 (size=43) 2024-11-26T15:29:06,881 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:06,882 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,884 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3bc0d04ec09592c2469db6cdfe741ec columnFamilyName a 2024-11-26T15:29:06,884 DEBUG [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:06,884 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(327): Store=e3bc0d04ec09592c2469db6cdfe741ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:06,884 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,886 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3bc0d04ec09592c2469db6cdfe741ec columnFamilyName b 2024-11-26T15:29:06,886 DEBUG [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:06,887 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(327): Store=e3bc0d04ec09592c2469db6cdfe741ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:06,887 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,888 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3bc0d04ec09592c2469db6cdfe741ec columnFamilyName c 2024-11-26T15:29:06,889 DEBUG [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:06,889 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(327): Store=e3bc0d04ec09592c2469db6cdfe741ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:06,889 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,890 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,892 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,893 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:06,894 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:06,897 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:06,898 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3bc0d04ec09592c2469db6cdfe741ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69344540, jitterRate=0.03331416845321655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:06,899 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3bc0d04ec09592c2469db6cdfe741ec: Writing region info on filesystem at 1732634946881Initializing all the Stores at 1732634946882 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634946882Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634946882Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634946882Cleaning up temporary data from old regions at 1732634946892 (+10 ms)Region opened successfully at 1732634946898 (+6 ms) 2024-11-26T15:29:06,899 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e3bc0d04ec09592c2469db6cdfe741ec, disabling compactions & flushes 2024-11-26T15:29:06,899 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:06,899 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:06,899 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. after waiting 0 ms 2024-11-26T15:29:06,899 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:06,899 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:06,899 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e3bc0d04ec09592c2469db6cdfe741ec: Waiting for close lock at 1732634946899Disabling compacts and flushes for region at 1732634946899Disabling writes for close at 1732634946899Writing region close event to WAL at 1732634946899Closed at 1732634946899 2024-11-26T15:29:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741864_1041 (size=93) 2024-11-26T15:29:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741864_1041 (size=93) 2024-11-26T15:29:06,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741864_1041 (size=93) 2024-11-26T15:29:06,907 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:06,907 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58018778:(num 1732634946850) 2024-11-26T15:29:06,908 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:06,910 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:06,925 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911, exclude list is [], retry=0 2024-11-26T15:29:06,928 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:06,928 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:06,929 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:06,931 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911 2024-11-26T15:29:06,931 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:07,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741866_1043 (size=344306) 2024-11-26T15:29:07,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741866_1043 (size=344306) 2024-11-26T15:29:07,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741866_1043 (size=344306) 2024-11-26T15:29:07,104 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911, size=336.2 K (344306bytes) 2024-11-26T15:29:07,104 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911 2024-11-26T15:29:07,104 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911 after 0ms 2024-11-26T15:29:07,108 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911 took 4ms 2024-11-26T15:29:07,113 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732634946911.temp 2024-11-26T15:29:07,115 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000000001-wal.1732634946911.temp 2024-11-26T15:29:07,139 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911 so closing down 2024-11-26T15:29:07,139 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:07,139 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741867_1044 (size=344306) 2024-11-26T15:29:07,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741867_1044 (size=344306) 2024-11-26T15:29:07,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741867_1044 (size=344306) 2024-11-26T15:29:07,151 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000000001-wal.1732634946911.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-26T15:29:07,152 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000000001-wal.1732634946911.temp to hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003000 2024-11-26T15:29:07,153 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-26T15:29:07,153 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911, journal: Splitting hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911, size=336.2 K (344306bytes) at 1732634947104Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000000001-wal.1732634946911.temp at 1732634947115 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732634947121 (+6 ms)Split 2048 edits, skipped 0 edits. at 1732634947131 (+10 ms)Finishing writing output for hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911 so closing down at 1732634947139 (+8 ms)3 split writer threads finished at 1732634947139Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000000001-wal.1732634946911.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1732634947151 (+12 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000000001-wal.1732634946911.temp to hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003000 at 1732634947152 (+1 ms)Processed 3000 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732634947153 (+1 ms) 2024-11-26T15:29:07,155 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634946911 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634946911 2024-11-26T15:29:07,156 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003000 2024-11-26T15:29:07,156 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:07,159 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:07,178 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159, exclude list is [], retry=0 2024-11-26T15:29:07,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:07,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:07,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:07,184 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159 2024-11-26T15:29:07,184 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:07,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741868_1045 (size=344433) 2024-11-26T15:29:07,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741868_1045 (size=344433) 2024-11-26T15:29:07,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741868_1045 (size=344433) 2024-11-26T15:29:07,341 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159, size=336.4 K (344433bytes) 2024-11-26T15:29:07,341 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159 2024-11-26T15:29:07,342 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159 after 1ms 2024-11-26T15:29:07,346 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159 took 5ms 2024-11-26T15:29:07,350 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732634947159.temp 2024-11-26T15:29:07,353 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003001-wal.1732634947159.temp 2024-11-26T15:29:07,379 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159 so closing down 2024-11-26T15:29:07,380 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:07,380 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:07,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741869_1046 (size=344433) 2024-11-26T15:29:07,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741869_1046 (size=344433) 2024-11-26T15:29:07,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741869_1046 (size=344433) 2024-11-26T15:29:07,385 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003001-wal.1732634947159.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-11-26T15:29:07,387 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003001-wal.1732634947159.temp to hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000006000 2024-11-26T15:29:07,387 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-26T15:29:07,387 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159, journal: Splitting hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159, size=336.4 K (344433bytes) at 1732634947341Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003001-wal.1732634947159.temp at 1732634947353 (+12 ms)Split 1024 edits, skipped 0 edits. at 1732634947359 (+6 ms)Split 2048 edits, skipped 0 edits. at 1732634947370 (+11 ms)Finishing writing output for hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159 so closing down at 1732634947380 (+10 ms)3 split writer threads finished at 1732634947380Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003001-wal.1732634947159.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1732634947385 (+5 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003001-wal.1732634947159.temp to hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000006000 at 1732634947387 (+2 ms)Processed 3000 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732634947387 2024-11-26T15:29:07,390 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947159 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634947159 2024-11-26T15:29:07,391 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000006000 2024-11-26T15:29:07,391 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:07,394 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/test2727-manual,16010,1732634946842, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:07,413 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947394, exclude list is [], retry=0 2024-11-26T15:29:07,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:07,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:07,417 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:07,419 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732634946842/wal.1732634947394 2024-11-26T15:29:07,420 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:07,420 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e3bc0d04ec09592c2469db6cdfe741ec, NAME => 'test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:07,420 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:07,420 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,420 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,424 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,426 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3bc0d04ec09592c2469db6cdfe741ec columnFamilyName a 2024-11-26T15:29:07,426 DEBUG [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:07,427 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(327): Store=e3bc0d04ec09592c2469db6cdfe741ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:07,427 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,428 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3bc0d04ec09592c2469db6cdfe741ec columnFamilyName b 2024-11-26T15:29:07,429 DEBUG [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:07,429 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(327): Store=e3bc0d04ec09592c2469db6cdfe741ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:07,429 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,431 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3bc0d04ec09592c2469db6cdfe741ec columnFamilyName c 2024-11-26T15:29:07,431 DEBUG [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:07,432 INFO [StoreOpener-e3bc0d04ec09592c2469db6cdfe741ec-1 {}] regionserver.HStore(327): Store=e3bc0d04ec09592c2469db6cdfe741ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:07,432 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,433 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,437 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003000 2024-11-26T15:29:07,473 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003000 2024-11-26T15:29:07,475 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000006000 2024-11-26T15:29:07,519 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000006000 2024-11-26T15:29:07,519 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e3bc0d04ec09592c2469db6cdfe741ec 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-26T15:29:07,541 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/a/240f978d7ff14abba48fc69d0c1d38ce is 41, key is test2727/a:100/1732634947188/Put/seqid=0 2024-11-26T15:29:07,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741871_1048 (size=84227) 2024-11-26T15:29:07,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741871_1048 (size=84227) 2024-11-26T15:29:07,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741871_1048 (size=84227) 2024-11-26T15:29:07,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/a/240f978d7ff14abba48fc69d0c1d38ce 2024-11-26T15:29:07,581 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/b/7a8881e4f31b4e4282b7077fa5b680d4 is 41, key is test2727/b:100/1732634947232/Put/seqid=0 2024-11-26T15:29:07,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741872_1049 (size=84609) 2024-11-26T15:29:07,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741872_1049 (size=84609) 2024-11-26T15:29:07,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741872_1049 (size=84609) 2024-11-26T15:29:07,588 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/b/7a8881e4f31b4e4282b7077fa5b680d4 2024-11-26T15:29:07,619 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/c/d619a3913ab84bc9920def73349ca39c is 41, key is test2727/c:100/1732634947274/Put/seqid=0 2024-11-26T15:29:07,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741873_1050 (size=84609) 2024-11-26T15:29:07,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741873_1050 (size=84609) 2024-11-26T15:29:07,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741873_1050 (size=84609) 2024-11-26T15:29:07,626 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/c/d619a3913ab84bc9920def73349ca39c 2024-11-26T15:29:07,635 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/a/240f978d7ff14abba48fc69d0c1d38ce as hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/a/240f978d7ff14abba48fc69d0c1d38ce 2024-11-26T15:29:07,643 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/a/240f978d7ff14abba48fc69d0c1d38ce, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-26T15:29:07,644 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/b/7a8881e4f31b4e4282b7077fa5b680d4 as hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/b/7a8881e4f31b4e4282b7077fa5b680d4 2024-11-26T15:29:07,652 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/b/7a8881e4f31b4e4282b7077fa5b680d4, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T15:29:07,653 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/.tmp/c/d619a3913ab84bc9920def73349ca39c as hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/c/d619a3913ab84bc9920def73349ca39c 2024-11-26T15:29:07,660 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/c/d619a3913ab84bc9920def73349ca39c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T15:29:07,661 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e3bc0d04ec09592c2469db6cdfe741ec in 141ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-26T15:29:07,662 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000003000 2024-11-26T15:29:07,662 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/0000000000000006000 2024-11-26T15:29:07,664 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,664 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,665 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:07,667 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3bc0d04ec09592c2469db6cdfe741ec 2024-11-26T15:29:07,669 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/test2727/e3bc0d04ec09592c2469db6cdfe741ec/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-26T15:29:07,670 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3bc0d04ec09592c2469db6cdfe741ec; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61945777, jitterRate=-0.07693599164485931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:07,672 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3bc0d04ec09592c2469db6cdfe741ec: Writing region info on filesystem at 1732634947421Initializing all the Stores at 1732634947422 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634947422Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634947424 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634947424Obtaining lock to block concurrent updates at 1732634947519 (+95 ms)Preparing flush snapshotting stores in e3bc0d04ec09592c2469db6cdfe741ec at 1732634947519Finished memstore snapshotting test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732634947519Flushing stores of test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. at 1732634947519Flushing e3bc0d04ec09592c2469db6cdfe741ec/a: creating writer at 1732634947519Flushing e3bc0d04ec09592c2469db6cdfe741ec/a: appending metadata at 1732634947540 (+21 ms)Flushing e3bc0d04ec09592c2469db6cdfe741ec/a: closing flushed file at 1732634947540Flushing e3bc0d04ec09592c2469db6cdfe741ec/b: creating writer at 1732634947559 (+19 ms)Flushing e3bc0d04ec09592c2469db6cdfe741ec/b: appending metadata at 1732634947579 (+20 ms)Flushing e3bc0d04ec09592c2469db6cdfe741ec/b: closing flushed file at 1732634947579Flushing e3bc0d04ec09592c2469db6cdfe741ec/c: creating writer at 1732634947596 (+17 ms)Flushing e3bc0d04ec09592c2469db6cdfe741ec/c: appending metadata at 1732634947618 (+22 ms)Flushing e3bc0d04ec09592c2469db6cdfe741ec/c: closing flushed file at 1732634947618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d1bb4f2: reopening flushed file at 1732634947634 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2532d956: reopening flushed file at 1732634947643 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f21e5a3: reopening flushed file at 1732634947652 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e3bc0d04ec09592c2469db6cdfe741ec in 141ms, sequenceid=6000, compaction requested=false; wal=null at 1732634947661 (+9 ms)Cleaning up temporary data from old regions at 1732634947664 (+3 ms)Region opened successfully at 1732634947671 (+7 ms) 2024-11-26T15:29:07,673 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-26T15:29:07,673 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e3bc0d04ec09592c2469db6cdfe741ec, disabling compactions & flushes 2024-11-26T15:29:07,673 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:07,673 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:07,673 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. after waiting 0 ms 2024-11-26T15:29:07,673 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:07,674 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732634946844.e3bc0d04ec09592c2469db6cdfe741ec. 2024-11-26T15:29:07,674 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e3bc0d04ec09592c2469db6cdfe741ec: Waiting for close lock at 1732634947673Disabling compacts and flushes for region at 1732634947673Disabling writes for close at 1732634947673Writing region close event to WAL at 1732634947674 (+1 ms)Closed at 1732634947674 2024-11-26T15:29:07,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741870_1047 (size=93) 2024-11-26T15:29:07,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741870_1047 (size=93) 2024-11-26T15:29:07,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741870_1047 (size=93) 2024-11-26T15:29:07,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:07,681 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732634947394) 2024-11-26T15:29:07,693 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=400 (was 398) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47798 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:33458 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:59612 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 255), ProcessCount=11 (was 11), AvailableMemoryMB=9529 (was 9721) 2024-11-26T15:29:07,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=400, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=9527 2024-11-26T15:29:07,721 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:07,728 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:07,729 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732634947728 2024-11-26T15:29:07,737 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728 2024-11-26T15:29:07,739 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:07,741 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 20c286dd973a211f43ea1a4a93be958f, NAME => 'testSequentialEditLogSeqNum,,1732634947722.20c286dd973a211f43ea1a4a93be958f.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:07,741 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732634947722.20c286dd973a211f43ea1a4a93be958f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:07,741 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,741 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,742 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f doesn't exist for region: 20c286dd973a211f43ea1a4a93be958f on table testSequentialEditLogSeqNum 2024-11-26T15:29:07,743 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 20c286dd973a211f43ea1a4a93be958f on table testSequentialEditLogSeqNum 2024-11-26T15:29:07,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741875_1052 (size=62) 2024-11-26T15:29:07,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741875_1052 (size=62) 2024-11-26T15:29:07,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741875_1052 (size=62) 2024-11-26T15:29:07,755 INFO [StoreOpener-20c286dd973a211f43ea1a4a93be958f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,756 INFO [StoreOpener-20c286dd973a211f43ea1a4a93be958f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20c286dd973a211f43ea1a4a93be958f columnFamilyName a 2024-11-26T15:29:07,756 DEBUG [StoreOpener-20c286dd973a211f43ea1a4a93be958f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:07,757 INFO [StoreOpener-20c286dd973a211f43ea1a4a93be958f-1 {}] regionserver.HStore(327): Store=20c286dd973a211f43ea1a4a93be958f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:07,757 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,758 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,758 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,758 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,758 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,760 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20c286dd973a211f43ea1a4a93be958f 2024-11-26T15:29:07,762 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:07,763 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20c286dd973a211f43ea1a4a93be958f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73842137, jitterRate=0.10033358633518219}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:07,764 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20c286dd973a211f43ea1a4a93be958f: Writing region info on filesystem at 1732634947741Initializing all the Stores at 1732634947754 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634947754Cleaning up temporary data from old regions at 1732634947758 (+4 ms)Region opened successfully at 1732634947764 (+6 ms) 2024-11-26T15:29:07,777 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 20c286dd973a211f43ea1a4a93be958f 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-26T15:29:07,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/.tmp/a/72ee536e5b3944c0b3db6c4e5334494a is 81, key is testSequentialEditLogSeqNum/a:x0/1732634947764/Put/seqid=0 2024-11-26T15:29:07,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741876_1053 (size=5833) 2024-11-26T15:29:07,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741876_1053 (size=5833) 2024-11-26T15:29:07,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741876_1053 (size=5833) 2024-11-26T15:29:07,808 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/.tmp/a/72ee536e5b3944c0b3db6c4e5334494a 2024-11-26T15:29:07,816 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T15:29:07,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:07,818 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/.tmp/a/72ee536e5b3944c0b3db6c4e5334494a as hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/a/72ee536e5b3944c0b3db6c4e5334494a 2024-11-26T15:29:07,819 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T15:29:07,819 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T15:29:07,819 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T15:29:07,820 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:07,821 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-26T15:29:07,821 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:07,822 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-26T15:29:07,822 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:07,827 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/a/72ee536e5b3944c0b3db6c4e5334494a, entries=10, sequenceid=13, filesize=5.7 K 2024-11-26T15:29:07,829 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 20c286dd973a211f43ea1a4a93be958f in 53ms, sequenceid=13, compaction requested=false 2024-11-26T15:29:07,829 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 20c286dd973a211f43ea1a4a93be958f: 2024-11-26T15:29:07,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T15:29:07,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T15:29:07,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T15:29:07,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T15:29:07,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T15:29:07,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741874_1051 (size=3017) 2024-11-26T15:29:07,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741874_1051 (size=3017) 2024-11-26T15:29:07,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741874_1051 (size=3017) 2024-11-26T15:29:07,865 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728, size=2.9 K (3017bytes) 2024-11-26T15:29:07,865 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728 2024-11-26T15:29:07,866 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728 after 0ms 2024-11-26T15:29:07,869 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728 took 5ms 2024-11-26T15:29:07,872 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728 so closing down 2024-11-26T15:29:07,872 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:07,873 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732634947728.temp 2024-11-26T15:29:07,875 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000003-wal.1732634947728.temp 2024-11-26T15:29:07,875 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:07,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741877_1054 (size=2433) 2024-11-26T15:29:07,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741877_1054 (size=2433) 2024-11-26T15:29:07,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741877_1054 (size=2433) 2024-11-26T15:29:07,885 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000003-wal.1732634947728.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:07,887 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000003-wal.1732634947728.temp to hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000020 2024-11-26T15:29:07,887 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-26T15:29:07,888 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728, journal: Splitting hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728, size=2.9 K (3017bytes) at 1732634947865Finishing writing output for hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728 so closing down at 1732634947872 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000003-wal.1732634947728.temp at 1732634947875 (+3 ms)3 split writer threads finished at 1732634947875Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000003-wal.1732634947728.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732634947885 (+10 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000003-wal.1732634947728.temp to hdfs://localhost:37937/hbase/data/default/testSequentialEditLogSeqNum/20c286dd973a211f43ea1a4a93be958f/recovered.edits/0000000000000000020 at 1732634947887 (+2 ms)Processed 17 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37937/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732634947720/wal.1732634947728, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732634947888 (+1 ms) 2024-11-26T15:29:07,901 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=405 (was 400) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47798 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:59612 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 255), ProcessCount=11 (was 11), AvailableMemoryMB=9510 (was 9527) 2024-11-26T15:29:07,914 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=405, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=9509 2024-11-26T15:29:07,931 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:07,933 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:07,961 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:07,964 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36317139, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-36317139, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:07,978 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36317139/hregion-36317139.1732634947965, exclude list is [], retry=0 2024-11-26T15:29:07,982 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:07,984 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:07,985 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:07,988 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36317139/hregion-36317139.1732634947965 2024-11-26T15:29:07,988 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:07,988 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a14d3df945a60af013da8f3b4952d7c3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741879_1056 (size=70) 2024-11-26T15:29:08,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741879_1056 (size=70) 2024-11-26T15:29:08,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741879_1056 (size=70) 2024-11-26T15:29:08,004 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:08,005 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,007 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName a 2024-11-26T15:29:08,007 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:08,008 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:08,008 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,010 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName b 2024-11-26T15:29:08,010 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:08,011 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:08,011 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,013 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName c 2024-11-26T15:29:08,013 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:08,014 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:08,014 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,015 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,016 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,016 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,017 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:08,018 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,021 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:08,021 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a14d3df945a60af013da8f3b4952d7c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59238365, jitterRate=-0.11727957427501678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:08,022 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a14d3df945a60af013da8f3b4952d7c3: Writing region info on filesystem at 1732634948004Initializing all the Stores at 1732634948005 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634948005Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634948005Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634948005Cleaning up temporary data from old regions at 1732634948016 (+11 ms)Region opened successfully at 1732634948022 (+6 ms) 2024-11-26T15:29:08,022 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a14d3df945a60af013da8f3b4952d7c3, disabling compactions & flushes 2024-11-26T15:29:08,022 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:08,023 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:08,023 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. after waiting 0 ms 2024-11-26T15:29:08,023 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:08,023 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:08,023 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a14d3df945a60af013da8f3b4952d7c3: Waiting for close lock at 1732634948022Disabling compacts and flushes for region at 1732634948022Disabling writes for close at 1732634948023 (+1 ms)Writing region close event to WAL at 1732634948023Closed at 1732634948023 2024-11-26T15:29:08,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741878_1055 (size=93) 2024-11-26T15:29:08,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741878_1055 (size=93) 2024-11-26T15:29:08,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741878_1055 (size=93) 2024-11-26T15:29:08,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:08,030 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36317139:(num 1732634947965) 2024-11-26T15:29:08,031 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:08,033 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:08,047 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034, exclude list is [], retry=0 2024-11-26T15:29:08,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:08,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:08,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:08,054 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 2024-11-26T15:29:08,054 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:08,055 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a14d3df945a60af013da8f3b4952d7c3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:08,055 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:08,055 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,055 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,057 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,058 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName a 2024-11-26T15:29:08,058 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:08,059 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:08,059 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,060 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName b 2024-11-26T15:29:08,060 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:08,061 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:08,061 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,062 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName c 2024-11-26T15:29:08,062 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:08,063 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:08,063 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,067 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,067 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,068 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:08,069 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:08,070 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a14d3df945a60af013da8f3b4952d7c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59934308, jitterRate=-0.10690921545028687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:08,071 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a14d3df945a60af013da8f3b4952d7c3: Writing region info on filesystem at 1732634948055Initializing all the Stores at 1732634948056 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634948056Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634948057 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634948057Cleaning up temporary data from old regions at 1732634948067 (+10 ms)Region opened successfully at 1732634948071 (+4 ms) 2024-11-26T15:29:08,076 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732634948075/Put/seqid=0 2024-11-26T15:29:08,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741881_1058 (size=4826) 2024-11-26T15:29:08,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741881_1058 (size=4826) 2024-11-26T15:29:08,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741881_1058 (size=4826) 2024-11-26T15:29:08,089 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37937/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in a14d3df945a60af013da8f3b4952d7c3/a 2024-11-26T15:29:08,098 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-26T15:29:08,098 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T15:29:08,098 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a14d3df945a60af013da8f3b4952d7c3: 2024-11-26T15:29:08,101 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/a/1bf027f11b99485f967334ba11044d4f_SeqId_3_ 2024-11-26T15:29:08,102 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37937/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a14d3df945a60af013da8f3b4952d7c3/a as hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/a/1bf027f11b99485f967334ba11044d4f_SeqId_3_ - updating store file list. 2024-11-26T15:29:08,108 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1bf027f11b99485f967334ba11044d4f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:08,108 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/a/1bf027f11b99485f967334ba11044d4f_SeqId_3_ into a14d3df945a60af013da8f3b4952d7c3/a 2024-11-26T15:29:08,109 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37937/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into a14d3df945a60af013da8f3b4952d7c3/a (new location: hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/a/1bf027f11b99485f967334ba11044d4f_SeqId_3_) 2024-11-26T15:29:08,154 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034, size=0 (0bytes) 2024-11-26T15:29:08,154 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 might be still open, length is 0 2024-11-26T15:29:08,154 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 2024-11-26T15:29:08,154 WARN [IPC Server handler 2 on default port 37937 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-26T15:29:08,155 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 after 1ms 2024-11-26T15:29:08,623 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:33532 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35535:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33532 dst: /127.0.0.1:35535 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35535 remote=/127.0.0.1:33532]. Total timeout mills is 60000, 59492 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:08,624 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47902 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34329:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47902 dst: /127.0.0.1:34329 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:08,624 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:59822 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39747:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59822 dst: /127.0.0.1:39747 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:08,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741880_1059 (size=504) 2024-11-26T15:29:08,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741880_1059 (size=504) 2024-11-26T15:29:12,156 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 after 4002ms 2024-11-26T15:29:12,164 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 took 4011ms 2024-11-26T15:29:12,168 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034; continuing. 2024-11-26T15:29:12,168 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 so closing down 2024-11-26T15:29:12,168 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:12,171 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732634948034.temp 2024-11-26T15:29:12,173 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005-wal.1732634948034.temp 2024-11-26T15:29:12,173 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:12,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741882_1060 (size=238) 2024-11-26T15:29:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741882_1060 (size=238) 2024-11-26T15:29:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741882_1060 (size=238) 2024-11-26T15:29:12,180 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005-wal.1732634948034.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:12,182 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005-wal.1732634948034.temp to hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005 2024-11-26T15:29:12,182 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T15:29:12,183 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034, journal: Splitting hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034, size=0 (0bytes) at 1732634948154Finishing writing output for hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 so closing down at 1732634952168 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005-wal.1732634948034.temp at 1732634952173 (+5 ms)3 split writer threads finished at 1732634952173Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005-wal.1732634948034.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732634952180 (+7 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005-wal.1732634948034.temp to hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005 at 1732634952182 (+2 ms)Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034, size=0, length=0, corrupted=false, cancelled=false at 1732634952182 2024-11-26T15:29:12,185 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634948034 2024-11-26T15:29:12,186 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005 2024-11-26T15:29:12,186 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:12,188 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:12,205 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634952189, exclude list is [], retry=0 2024-11-26T15:29:12,207 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:12,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:12,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:12,210 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634952189 2024-11-26T15:29:12,210 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:12,210 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a14d3df945a60af013da8f3b4952d7c3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:12,211 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:12,211 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,211 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,212 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,213 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName a 2024-11-26T15:29:12,213 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:12,219 DEBUG [StoreFileOpener-a14d3df945a60af013da8f3b4952d7c3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1bf027f11b99485f967334ba11044d4f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:12,220 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/a/1bf027f11b99485f967334ba11044d4f_SeqId_3_ 2024-11-26T15:29:12,220 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:12,220 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,221 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName b 2024-11-26T15:29:12,221 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:12,222 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:12,222 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,223 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a14d3df945a60af013da8f3b4952d7c3 columnFamilyName c 2024-11-26T15:29:12,223 DEBUG [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:12,223 INFO [StoreOpener-a14d3df945a60af013da8f3b4952d7c3-1 {}] regionserver.HStore(327): Store=a14d3df945a60af013da8f3b4952d7c3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:12,224 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,226 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,227 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005 2024-11-26T15:29:12,231 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005 2024-11-26T15:29:12,231 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a14d3df945a60af013da8f3b4952d7c3 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-26T15:29:12,248 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/.tmp/a/96cfff15e4504b708456313d24931b6a is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732634948114/Put/seqid=0 2024-11-26T15:29:12,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741884_1062 (size=5149) 2024-11-26T15:29:12,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741884_1062 (size=5149) 2024-11-26T15:29:12,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741884_1062 (size=5149) 2024-11-26T15:29:12,256 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/.tmp/a/96cfff15e4504b708456313d24931b6a 2024-11-26T15:29:12,263 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/.tmp/a/96cfff15e4504b708456313d24931b6a as hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/a/96cfff15e4504b708456313d24931b6a 2024-11-26T15:29:12,289 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/a/96cfff15e4504b708456313d24931b6a, entries=1, sequenceid=5, filesize=5.0 K 2024-11-26T15:29:12,290 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a14d3df945a60af013da8f3b4952d7c3 in 59ms, sequenceid=5, compaction requested=false; wal=null 2024-11-26T15:29:12,291 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/0000000000000000005 2024-11-26T15:29:12,292 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,292 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,293 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:12,294 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a14d3df945a60af013da8f3b4952d7c3 2024-11-26T15:29:12,296 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/a14d3df945a60af013da8f3b4952d7c3/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-26T15:29:12,298 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a14d3df945a60af013da8f3b4952d7c3; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65345211, jitterRate=-0.026280477643013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:12,298 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a14d3df945a60af013da8f3b4952d7c3: Writing region info on filesystem at 1732634952211Initializing all the Stores at 1732634952212 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634952212Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634952212Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634952212Obtaining lock to block concurrent updates at 1732634952231 (+19 ms)Preparing flush snapshotting stores in a14d3df945a60af013da8f3b4952d7c3 at 1732634952231Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732634952231Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. at 1732634952231Flushing a14d3df945a60af013da8f3b4952d7c3/a: creating writer at 1732634952231Flushing a14d3df945a60af013da8f3b4952d7c3/a: appending metadata at 1732634952248 (+17 ms)Flushing a14d3df945a60af013da8f3b4952d7c3/a: closing flushed file at 1732634952248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73a1d08c: reopening flushed file at 1732634952262 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for a14d3df945a60af013da8f3b4952d7c3 in 59ms, sequenceid=5, compaction requested=false; wal=null at 1732634952290 (+28 ms)Cleaning up temporary data from old regions at 1732634952292 (+2 ms)Region opened successfully at 1732634952298 (+6 ms) 2024-11-26T15:29:12,302 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a14d3df945a60af013da8f3b4952d7c3, disabling compactions & flushes 2024-11-26T15:29:12,302 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:12,302 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:12,303 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. after waiting 0 ms 2024-11-26T15:29:12,303 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:12,303 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732634947932.a14d3df945a60af013da8f3b4952d7c3. 2024-11-26T15:29:12,303 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a14d3df945a60af013da8f3b4952d7c3: Waiting for close lock at 1732634952302Disabling compacts and flushes for region at 1732634952302Disabling writes for close at 1732634952303 (+1 ms)Writing region close event to WAL at 1732634952303Closed at 1732634952303 2024-11-26T15:29:12,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741883_1061 (size=93) 2024-11-26T15:29:12,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741883_1061 (size=93) 2024-11-26T15:29:12,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741883_1061 (size=93) 2024-11-26T15:29:12,309 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:12,309 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732634952189) 2024-11-26T15:29:12,329 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 405) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:37937 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:37937 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-646493456_22 at /127.0.0.1:45798 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-646493456_22 at /127.0.0.1:60376 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-646493456_22 at /127.0.0.1:37392 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 255), ProcessCount=11 (was 11), AvailableMemoryMB=9427 (was 9509) 2024-11-26T15:29:12,340 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=9426 2024-11-26T15:29:12,356 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:12,360 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T15:29:12,364 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6cc2bc8bf733,44737,1732634937484 2024-11-26T15:29:12,366 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@628918b3 2024-11-26T15:29:12,367 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T15:29:12,369 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T15:29:12,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T15:29:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T15:29:12,380 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T15:29:12,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-26T15:29:12,382 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:12,384 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T15:29:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:29:12,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741885_1063 (size=694) 2024-11-26T15:29:12,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741885_1063 (size=694) 2024-11-26T15:29:12,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741885_1063 (size=694) 2024-11-26T15:29:12,397 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1122cfb9a9b693d960440938ac541ca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7 2024-11-26T15:29:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741886_1064 (size=77) 2024-11-26T15:29:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741886_1064 (size=77) 2024-11-26T15:29:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741886_1064 (size=77) 2024-11-26T15:29:12,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:12,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 1122cfb9a9b693d960440938ac541ca4, disabling compactions & flushes 2024-11-26T15:29:12,406 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:12,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:12,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. after waiting 0 ms 2024-11-26T15:29:12,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:12,406 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:12,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1122cfb9a9b693d960440938ac541ca4: Waiting for close lock at 1732634952406Disabling compacts and flushes for region at 1732634952406Disabling writes for close at 1732634952406Writing region close event to WAL at 1732634952406Closed at 1732634952406 2024-11-26T15:29:12,408 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T15:29:12,414 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732634952408"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732634952408"}]},"ts":"1732634952408"} 2024-11-26T15:29:12,418 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T15:29:12,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T15:29:12,423 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634952420"}]},"ts":"1732634952420"} 2024-11-26T15:29:12,427 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-26T15:29:12,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {6cc2bc8bf733=0} racks are {/default-rack=0} 2024-11-26T15:29:12,429 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T15:29:12,429 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T15:29:12,429 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T15:29:12,429 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T15:29:12,429 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T15:29:12,429 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T15:29:12,429 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T15:29:12,429 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T15:29:12,429 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T15:29:12,429 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T15:29:12,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN}] 2024-11-26T15:29:12,433 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN 2024-11-26T15:29:12,435 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN; state=OFFLINE, location=6cc2bc8bf733,43161,1732634938266; forceNewPlan=false, retain=false 2024-11-26T15:29:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:29:12,588 INFO [6cc2bc8bf733:44737 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T15:29:12,589 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPENING, regionLocation=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:12,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN because future has completed 2024-11-26T15:29:12,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266}] 2024-11-26T15:29:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:29:12,755 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:12,756 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1122cfb9a9b693d960440938ac541ca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:12,756 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,757 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:12,757 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,757 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,760 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,762 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf1 2024-11-26T15:29:12,763 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:12,763 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:12,764 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,765 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf2 2024-11-26T15:29:12,765 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:12,766 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:12,766 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,767 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,768 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,769 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,769 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,771 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T15:29:12,773 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,775 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:12,776 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1122cfb9a9b693d960440938ac541ca4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69957928, jitterRate=0.04245436191558838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T15:29:12,776 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:12,778 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1122cfb9a9b693d960440938ac541ca4: Running coprocessor pre-open hook at 1732634952757Writing region info on filesystem at 1732634952757Initializing all the Stores at 1732634952759 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634952759Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634952760 (+1 ms)Cleaning up temporary data from old regions at 1732634952769 (+9 ms)Running coprocessor post-open hooks at 1732634952777 (+8 ms)Region opened successfully at 1732634952778 (+1 ms) 2024-11-26T15:29:12,779 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., pid=6, masterSystemTime=1732634952746 2024-11-26T15:29:12,783 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:12,783 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:12,784 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPEN, openSeqNum=2, regionLocation=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:12,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266 because future has completed 2024-11-26T15:29:12,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T15:29:12,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266 in 195 msec 2024-11-26T15:29:12,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T15:29:12,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN in 360 msec 2024-11-26T15:29:12,795 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T15:29:12,796 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732634952795"}]},"ts":"1732634952795"} 2024-11-26T15:29:12,798 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-26T15:29:12,800 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T15:29:12,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 426 msec 2024-11-26T15:29:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T15:29:13,022 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-26T15:29:13,023 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-26T15:29:13,025 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T15:29:13,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-26T15:29:13,035 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T15:29:13,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-26T15:29:13,053 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=2] 2024-11-26T15:29:13,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=1122cfb9a9b693d960440938ac541ca4, source=6cc2bc8bf733,43161,1732634938266, destination=6cc2bc8bf733,41249,1732634938354, warming up region on 6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:13,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T15:29:13,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=1122cfb9a9b693d960440938ac541ca4, source=6cc2bc8bf733,43161,1732634938266, destination=6cc2bc8bf733,41249,1732634938354, running balancer 2024-11-26T15:29:13,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46651, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T15:29:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE 2024-11-26T15:29:13,074 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE 2024-11-26T15:29:13,076 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=CLOSING, regionLocation=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:13,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1122cfb9a9b693d960440938ac541ca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:13,079 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE because future has completed 2024-11-26T15:29:13,080 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:29:13,080 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf1 2024-11-26T15:29:13,080 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:13,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266}] 2024-11-26T15:29:13,081 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:13,081 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,082 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf2 2024-11-26T15:29:13,083 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:13,083 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(1722): Closing 1122cfb9a9b693d960440938ac541ca4, disabling compactions & flushes 2024-11-26T15:29:13,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. after waiting 0 ms 2024-11-26T15:29:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] regionserver.HRegion(1676): Region close journal for 1122cfb9a9b693d960440938ac541ca4: Waiting for close lock at 1732634953084Disabling compacts and flushes for region at 1732634953084Disabling writes for close at 1732634953084Writing region close event to WAL at 1732634953084Closed at 1732634953084 2024-11-26T15:29:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T15:29:13,240 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,240 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:29:13,240 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1122cfb9a9b693d960440938ac541ca4, disabling compactions & flushes 2024-11-26T15:29:13,240 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,241 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,241 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. after waiting 0 ms 2024-11-26T15:29:13,241 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,241 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 1122cfb9a9b693d960440938ac541ca4 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-26T15:29:13,257 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf1/f3c7790baf2a459ead6203cee7cf4242 is 35, key is r1/cf1:q/1732634953056/Put/seqid=0 2024-11-26T15:29:13,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741887_1065 (size=4783) 2024-11-26T15:29:13,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741887_1065 (size=4783) 2024-11-26T15:29:13,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741887_1065 (size=4783) 2024-11-26T15:29:13,265 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf1/f3c7790baf2a459ead6203cee7cf4242 2024-11-26T15:29:13,272 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf1/f3c7790baf2a459ead6203cee7cf4242 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242 2024-11-26T15:29:13,279 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242, entries=1, sequenceid=5, filesize=4.7 K 2024-11-26T15:29:13,280 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1122cfb9a9b693d960440938ac541ca4 in 39ms, sequenceid=5, compaction requested=false 2024-11-26T15:29:13,280 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-26T15:29:13,286 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T15:29:13,289 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,289 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1122cfb9a9b693d960440938ac541ca4: Waiting for close lock at 1732634953240Running coprocessor pre-close hooks at 1732634953240Disabling compacts and flushes for region at 1732634953240Disabling writes for close at 1732634953241 (+1 ms)Obtaining lock to block concurrent updates at 1732634953241Preparing flush snapshotting stores in 1122cfb9a9b693d960440938ac541ca4 at 1732634953241Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732634953241Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. at 1732634953242 (+1 ms)Flushing 1122cfb9a9b693d960440938ac541ca4/cf1: creating writer at 1732634953242Flushing 1122cfb9a9b693d960440938ac541ca4/cf1: appending metadata at 1732634953257 (+15 ms)Flushing 1122cfb9a9b693d960440938ac541ca4/cf1: closing flushed file at 1732634953257Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@223ef619: reopening flushed file at 1732634953271 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1122cfb9a9b693d960440938ac541ca4 in 39ms, sequenceid=5, compaction requested=false at 1732634953280 (+9 ms)Writing region close event to WAL at 1732634953281 (+1 ms)Running coprocessor post-close hooks at 1732634953287 (+6 ms)Closed at 1732634953289 (+2 ms) 2024-11-26T15:29:13,289 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1122cfb9a9b693d960440938ac541ca4 move to 6cc2bc8bf733,41249,1732634938354 record at close sequenceid=5 2024-11-26T15:29:13,293 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,294 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=CLOSED 2024-11-26T15:29:13,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266 because future has completed 2024-11-26T15:29:13,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T15:29:13,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266 in 220 msec 2024-11-26T15:29:13,305 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE; state=CLOSED, location=6cc2bc8bf733,41249,1732634938354; forceNewPlan=false, retain=false 2024-11-26T15:29:13,456 INFO [6cc2bc8bf733:44737 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T15:29:13,456 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPENING, regionLocation=6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:13,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE because future has completed 2024-11-26T15:29:13,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354}] 2024-11-26T15:29:13,624 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,624 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1122cfb9a9b693d960440938ac541ca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:13,625 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,625 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:13,625 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,626 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,629 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,630 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf1 2024-11-26T15:29:13,630 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:13,640 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242 2024-11-26T15:29:13,640 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:13,640 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,641 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf2 2024-11-26T15:29:13,641 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:13,642 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:13,642 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,643 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,644 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,644 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,644 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,645 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T15:29:13,647 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,647 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1122cfb9a9b693d960440938ac541ca4; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65099046, jitterRate=-0.02994862198829651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T15:29:13,647 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:13,648 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1122cfb9a9b693d960440938ac541ca4: Running coprocessor pre-open hook at 1732634953626Writing region info on filesystem at 1732634953626Initializing all the Stores at 1732634953628 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634953628Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634953628Cleaning up temporary data from old regions at 1732634953644 (+16 ms)Running coprocessor post-open hooks at 1732634953648 (+4 ms)Region opened successfully at 1732634953648 2024-11-26T15:29:13,649 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., pid=9, masterSystemTime=1732634953616 2024-11-26T15:29:13,652 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,652 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,653 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPEN, openSeqNum=9, regionLocation=6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:13,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354 because future has completed 2024-11-26T15:29:13,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T15:29:13,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354 in 194 msec 2024-11-26T15:29:13,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE in 588 msec 2024-11-26T15:29:13,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T15:29:13,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T15:29:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:34356 deadline: 1732635013687, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=41249 startCode=1732634938354. As of locationSeqNum=5. 2024-11-26T15:29:13,709 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=41249 startCode=1732634938354. As of locationSeqNum=5. 2024-11-26T15:29:13,709 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=41249 startCode=1732634938354. As of locationSeqNum=5. 2024-11-26T15:29:13,709 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,41249,1732634938354, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=41249 startCode=1732634938354. As of locationSeqNum=5. 2024-11-26T15:29:13,821 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:29:13,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:29:13,831 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1122cfb9a9b693d960440938ac541ca4 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-26T15:29:13,849 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf1/545badb4422d403e9a0b510f312015f2 is 29, key is r1/cf1:/1732634953824/DeleteFamily/seqid=0 2024-11-26T15:29:13,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741888_1066 (size=4906) 2024-11-26T15:29:13,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741888_1066 (size=4906) 2024-11-26T15:29:13,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741888_1066 (size=4906) 2024-11-26T15:29:13,857 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf1/545badb4422d403e9a0b510f312015f2 2024-11-26T15:29:13,863 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 545badb4422d403e9a0b510f312015f2 2024-11-26T15:29:13,878 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501 is 29, key is r1/cf2:/1732634953824/DeleteFamily/seqid=0 2024-11-26T15:29:13,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741889_1067 (size=4906) 2024-11-26T15:29:13,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741889_1067 (size=4906) 2024-11-26T15:29:13,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741889_1067 (size=4906) 2024-11-26T15:29:13,886 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501 2024-11-26T15:29:13,894 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b9be342f7e0b4a7fbc90ce66dbcbd501 2024-11-26T15:29:13,895 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf1/545badb4422d403e9a0b510f312015f2 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/545badb4422d403e9a0b510f312015f2 2024-11-26T15:29:13,902 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 545badb4422d403e9a0b510f312015f2 2024-11-26T15:29:13,903 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/545badb4422d403e9a0b510f312015f2, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T15:29:13,904 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501 2024-11-26T15:29:13,911 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b9be342f7e0b4a7fbc90ce66dbcbd501 2024-11-26T15:29:13,911 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T15:29:13,912 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 1122cfb9a9b693d960440938ac541ca4 in 81ms, sequenceid=12, compaction requested=false 2024-11-26T15:29:13,912 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1122cfb9a9b693d960440938ac541ca4: 2024-11-26T15:29:13,915 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T15:29:13,916 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1122cfb9a9b693d960440938ac541ca4/cf1 is initiating major compaction (all files) 2024-11-26T15:29:13,916 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T15:29:13,916 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:13,917 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1122cfb9a9b693d960440938ac541ca4/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,917 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242, hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/545badb4422d403e9a0b510f312015f2] into tmpdir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp, totalSize=9.5 K 2024-11-26T15:29:13,918 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f3c7790baf2a459ead6203cee7cf4242, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732634953056 2024-11-26T15:29:13,919 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 545badb4422d403e9a0b510f312015f2, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T15:29:13,931 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1122cfb9a9b693d960440938ac541ca4#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:29:13,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741890_1068 (size=4626) 2024-11-26T15:29:13,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741890_1068 (size=4626) 2024-11-26T15:29:13,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741890_1068 (size=4626) 2024-11-26T15:29:13,945 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf1/785b5f9d8e714a39a99954502d7e3003 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/785b5f9d8e714a39a99954502d7e3003 2024-11-26T15:29:13,959 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 1122cfb9a9b693d960440938ac541ca4/cf1 of 1122cfb9a9b693d960440938ac541ca4 into 785b5f9d8e714a39a99954502d7e3003(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:29:13,959 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1122cfb9a9b693d960440938ac541ca4: 2024-11-26T15:29:13,959 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-26T15:29:13,959 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1122cfb9a9b693d960440938ac541ca4/cf2 is initiating major compaction (all files) 2024-11-26T15:29:13,959 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T15:29:13,959 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T15:29:13,960 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1122cfb9a9b693d960440938ac541ca4/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:13,960 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501] into tmpdir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp, totalSize=4.8 K 2024-11-26T15:29:13,960 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b9be342f7e0b4a7fbc90ce66dbcbd501, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T15:29:13,966 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1122cfb9a9b693d960440938ac541ca4#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T15:29:13,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741891_1069 (size=4592) 2024-11-26T15:29:13,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741891_1069 (size=4592) 2024-11-26T15:29:13,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741891_1069 (size=4592) 2024-11-26T15:29:13,981 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/.tmp/cf2/890e6a5d7e9444d38470806337b23970 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/890e6a5d7e9444d38470806337b23970 2024-11-26T15:29:13,989 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 1122cfb9a9b693d960440938ac541ca4/cf2 of 1122cfb9a9b693d960440938ac541ca4 into 890e6a5d7e9444d38470806337b23970(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:29:13,989 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1122cfb9a9b693d960440938ac541ca4: 2024-11-26T15:29:13,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=1122cfb9a9b693d960440938ac541ca4, source=6cc2bc8bf733,41249,1732634938354, destination=6cc2bc8bf733,43161,1732634938266, warming up region on 6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:13,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=1122cfb9a9b693d960440938ac541ca4, source=6cc2bc8bf733,41249,1732634938354, destination=6cc2bc8bf733,43161,1732634938266, running balancer 2024-11-26T15:29:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE 2024-11-26T15:29:13,998 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE 2024-11-26T15:29:14,000 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=CLOSING, regionLocation=6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:14,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1122cfb9a9b693d960440938ac541ca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:14,004 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE because future has completed 2024-11-26T15:29:14,004 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T15:29:14,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354}] 2024-11-26T15:29:14,005 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf1 2024-11-26T15:29:14,005 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:14,012 INFO [StoreFileOpener-1122cfb9a9b693d960440938ac541ca4-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 545badb4422d403e9a0b510f312015f2 2024-11-26T15:29:14,013 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/545badb4422d403e9a0b510f312015f2 2024-11-26T15:29:14,018 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/785b5f9d8e714a39a99954502d7e3003 2024-11-26T15:29:14,023 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242 2024-11-26T15:29:14,023 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:14,023 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,024 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf2 2024-11-26T15:29:14,024 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:14,030 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/890e6a5d7e9444d38470806337b23970 2024-11-26T15:29:14,035 INFO [StoreFileOpener-1122cfb9a9b693d960440938ac541ca4-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b9be342f7e0b4a7fbc90ce66dbcbd501 2024-11-26T15:29:14,035 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501 2024-11-26T15:29:14,035 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(1722): Closing 1122cfb9a9b693d960440938ac541ca4, disabling compactions & flushes 2024-11-26T15:29:14,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. after waiting 0 ms 2024-11-26T15:29:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43161 {}] regionserver.HRegion(1676): Region close journal for 1122cfb9a9b693d960440938ac541ca4: Waiting for close lock at 1732634954035Disabling compacts and flushes for region at 1732634954035Disabling writes for close at 1732634954036 (+1 ms)Writing region close event to WAL at 1732634954036Closed at 1732634954036 2024-11-26T15:29:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-26T15:29:14,159 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,159 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T15:29:14,159 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 1122cfb9a9b693d960440938ac541ca4, disabling compactions & flushes 2024-11-26T15:29:14,159 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,159 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,159 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. after waiting 0 ms 2024-11-26T15:29:14,159 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,160 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242, hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/545badb4422d403e9a0b510f312015f2] to archive 2024-11-26T15:29:14,162 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:29:14,166 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242 to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/f3c7790baf2a459ead6203cee7cf4242 2024-11-26T15:29:14,167 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/545badb4422d403e9a0b510f312015f2 to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/545badb4422d403e9a0b510f312015f2 2024-11-26T15:29:14,179 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501] to archive 2024-11-26T15:29:14,180 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:29:14,183 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501 to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/b9be342f7e0b4a7fbc90ce66dbcbd501 2024-11-26T15:29:14,189 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-26T15:29:14,189 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,189 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 1122cfb9a9b693d960440938ac541ca4: Waiting for close lock at 1732634954159Running coprocessor pre-close hooks at 1732634954159Disabling compacts and flushes for region at 1732634954159Disabling writes for close at 1732634954159Writing region close event to WAL at 1732634954185 (+26 ms)Running coprocessor post-close hooks at 1732634954189 (+4 ms)Closed at 1732634954189 2024-11-26T15:29:14,189 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 1122cfb9a9b693d960440938ac541ca4 move to 6cc2bc8bf733,43161,1732634938266 record at close sequenceid=12 2024-11-26T15:29:14,192 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,193 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=CLOSED 2024-11-26T15:29:14,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354 because future has completed 2024-11-26T15:29:14,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-26T15:29:14,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354 in 193 msec 2024-11-26T15:29:14,200 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE; state=CLOSED, location=6cc2bc8bf733,43161,1732634938266; forceNewPlan=false, retain=false 2024-11-26T15:29:14,351 INFO [6cc2bc8bf733:44737 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T15:29:14,351 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPENING, regionLocation=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:14,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE because future has completed 2024-11-26T15:29:14,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266}] 2024-11-26T15:29:14,522 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,522 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 1122cfb9a9b693d960440938ac541ca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:14,523 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,523 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:14,523 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,523 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,525 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,526 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf1 2024-11-26T15:29:14,526 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:14,533 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/785b5f9d8e714a39a99954502d7e3003 2024-11-26T15:29:14,533 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:14,533 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,534 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf2 2024-11-26T15:29:14,534 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:14,541 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/890e6a5d7e9444d38470806337b23970 2024-11-26T15:29:14,541 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:14,541 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,542 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,543 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,544 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,544 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,545 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T15:29:14,546 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,547 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 1122cfb9a9b693d960440938ac541ca4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61310989, jitterRate=-0.08639506995677948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T15:29:14,547 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,548 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 1122cfb9a9b693d960440938ac541ca4: Running coprocessor pre-open hook at 1732634954523Writing region info on filesystem at 1732634954523Initializing all the Stores at 1732634954524 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634954524Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634954525 (+1 ms)Cleaning up temporary data from old regions at 1732634954544 (+19 ms)Running coprocessor post-open hooks at 1732634954547 (+3 ms)Region opened successfully at 1732634954548 (+1 ms) 2024-11-26T15:29:14,549 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., pid=12, masterSystemTime=1732634954512 2024-11-26T15:29:14,551 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,551 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,552 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPEN, openSeqNum=18, regionLocation=6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:14,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266 because future has completed 2024-11-26T15:29:14,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-26T15:29:14,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,43161,1732634938266 in 196 msec 2024-11-26T15:29:14,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, REOPEN/MOVE in 560 msec 2024-11-26T15:29:14,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T15:29:14,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T15:29:14,604 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 6cc2bc8bf733,43161,1732634938266: testing ***** 2024-11-26T15:29:14,604 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T15:29:14,606 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T15:29:14,607 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T15:29:14,610 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T15:29:14,611 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T15:29:14,620 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 262349440 }, "NonHeapMemoryUsage": { "committed": 170131456, "init": 7667712, "max": -1, "used": 167555920 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "6cc2bc8bf733", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "6cc2bc8bf733", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60921", "tag.serverName": "6cc2bc8bf733,41249,1732634938354", "tag.clusterId": "b86a04a6-48ba-44c0-ade9-bd78fee725c1", "tag.Context": "regionserver", "tag.Hostname": "6cc2bc8bf733", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732634938354, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-26T15:29:14,624 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44737 {}] master.MasterRpcServices(700): 6cc2bc8bf733,43161,1732634938266 reported a fatal error: ***** ABORTING region server 6cc2bc8bf733,43161,1732634938266: testing ***** 2024-11-26T15:29:14,626 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6cc2bc8bf733,43161,1732634938266' ***** 2024-11-26T15:29:14,626 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-26T15:29:14,626 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T15:29:14,627 INFO [RS:0;6cc2bc8bf733:43161 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-26T15:29:14,627 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T15:29:14,627 INFO [RS:0;6cc2bc8bf733:43161 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-26T15:29:14,627 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(3091): Received CLOSE for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41249 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:35820 deadline: 1732635014627, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=43161 startCode=1732634938266. As of locationSeqNum=12. 2024-11-26T15:29:14,627 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(956): aborting server 6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:14,628 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T15:29:14,628 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1122cfb9a9b693d960440938ac541ca4, disabling compactions & flushes 2024-11-26T15:29:14,628 INFO [RS:0;6cc2bc8bf733:43161 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6cc2bc8bf733:43161. 2024-11-26T15:29:14,628 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,628 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,628 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,41249,1732634938354, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,41249,1732634938354, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=43161 startCode=1732634938266. As of locationSeqNum=12. 2024-11-26T15:29:14,628 DEBUG [RS:0;6cc2bc8bf733:43161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:29:14,628 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. after waiting 0 ms 2024-11-26T15:29:14,628 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,41249,1732634938354, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=43161 startCode=1732634938266. As of locationSeqNum=12. 2024-11-26T15:29:14,628 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,628 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,41249,1732634938354, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6cc2bc8bf733 port=43161 startCode=1732634938266. As of locationSeqNum=12. 2024-11-26T15:29:14,629 DEBUG [RS:0;6cc2bc8bf733:43161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:14,629 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T15:29:14,629 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T15:29:14,629 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T15:29:14,629 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T15:29:14,629 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T15:29:14,630 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T15:29:14,630 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1122cfb9a9b693d960440938ac541ca4=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.} 2024-11-26T15:29:14,630 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T15:29:14,630 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T15:29:14,630 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,630 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T15:29:14,630 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1122cfb9a9b693d960440938ac541ca4: Waiting for close lock at 1732634954628Running coprocessor pre-close hooks at 1732634954628Disabling compacts and flushes for region at 1732634954628Disabling writes for close at 1732634954628Writing region close event to WAL at 1732634954629 (+1 ms)Running coprocessor post-close hooks at 1732634954629Closed at 1732634954630 (+1 ms) 2024-11-26T15:29:14,630 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T15:29:14,630 DEBUG [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1351): Waiting on 1122cfb9a9b693d960440938ac541ca4, 1588230740 2024-11-26T15:29:14,630 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:14,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6cc2bc8bf733,43161,1732634938266 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:14,630 ERROR [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-26T15:29:14,630 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T15:29:14,631 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T15:29:14,631 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732634954629Running coprocessor pre-close hooks at 1732634954629Disabling compacts and flushes for region at 1732634954629Disabling writes for close at 1732634954630 (+1 ms)Writing region close event to WAL at 1732634954630Running coprocessor post-close hooks at 1732634954630Closed at 1732634954630 2024-11-26T15:29:14,631 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T15:29:14,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6cc2bc8bf733,43161,1732634938266 aborting 2024-11-26T15:29:14,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6cc2bc8bf733,43161,1732634938266 aborting 2024-11-26T15:29:14,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 from cache 2024-11-26T15:29:14,642 INFO [regionserver/6cc2bc8bf733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:29:14,722 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T15:29:14,722 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T15:29:14,739 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:29:14,741 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1] 2024-11-26T15:29:14,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6cc2bc8bf733,43161,1732634938266 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:14,743 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6cc2bc8bf733,43161,1732634938266 aborting 2024-11-26T15:29:14,743 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6cc2bc8bf733,43161,1732634938266 aborting 2024-11-26T15:29:14,743 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 from cache 2024-11-26T15:29:14,830 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(976): stopping server 6cc2bc8bf733,43161,1732634938266; all regions closed. 2024-11-26T15:29:14,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741836_1012 (size=6845) 2024-11-26T15:29:14,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741836_1012 (size=6845) 2024-11-26T15:29:14,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741836_1012 (size=6845) 2024-11-26T15:29:14,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741833_1009 (size=2142) 2024-11-26T15:29:14,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741833_1009 (size=2142) 2024-11-26T15:29:14,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741833_1009 (size=2142) 2024-11-26T15:29:14,839 DEBUG [RS:0;6cc2bc8bf733:43161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:14,839 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:29:14,839 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T15:29:14,839 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.ChoreService(370): Chore service for: regionserver/6cc2bc8bf733:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T15:29:14,840 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T15:29:14,840 INFO [regionserver/6cc2bc8bf733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T15:29:14,841 INFO [RS:0;6cc2bc8bf733:43161 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43161 2024-11-26T15:29:14,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T15:29:14,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6cc2bc8bf733,43161,1732634938266 2024-11-26T15:29:14,926 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T15:29:14,936 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6cc2bc8bf733,43161,1732634938266] 2024-11-26T15:29:14,946 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6cc2bc8bf733,43161,1732634938266 already deleted, retry=false 2024-11-26T15:29:14,947 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6cc2bc8bf733,43161,1732634938266 on 6cc2bc8bf733,44737,1732634937484 2024-11-26T15:29:14,950 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:29:14,952 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1] 2024-11-26T15:29:14,955 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6cc2bc8bf733:43161 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6cc2bc8bf733/172.17.0.3:43161 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:14,956 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6cc2bc8bf733,43161,1732634938266, splitWal=true, meta=true 2024-11-26T15:29:14,956 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1, error=java.net.ConnectException: Call to address=6cc2bc8bf733:43161 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6cc2bc8bf733/172.17.0.3:43161 2024-11-26T15:29:14,957 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-26T15:29:14,957 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 from cache 2024-11-26T15:29:14,958 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 6cc2bc8bf733:43161 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6cc2bc8bf733/172.17.0.3:43161 2024-11-26T15:29:14,959 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6cc2bc8bf733,43161,1732634938266, splitWal=true, meta=true 2024-11-26T15:29:14,959 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 6cc2bc8bf733,43161,1732634938266 (carryingMeta=true) 6cc2bc8bf733,43161,1732634938266/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4552327d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T15:29:14,961 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6cc2bc8bf733,43161,1732634938266, splitWal=true, meta=true, isMeta: true 2024-11-26T15:29:14,962 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting 2024-11-26T15:29:14,963 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6cc2bc8bf733,43161,1732634938266 WAL count=1, meta=true 2024-11-26T15:29:14,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta}] 2024-11-26T15:29:14,969 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:14,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, worker=6cc2bc8bf733,42689,1732634938394}] 2024-11-26T15:29:15,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:15,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43161-0x10177c301390001, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:15,037 INFO [RS:0;6cc2bc8bf733:43161 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T15:29:15,037 INFO [RS:0;6cc2bc8bf733:43161 {}] regionserver.HRegionServer(1031): Exiting; stopping=6cc2bc8bf733,43161,1732634938266; zookeeper connection closed. 2024-11-26T15:29:15,038 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7356fb74 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7356fb74 2024-11-26T15:29:15,142 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T15:29:15,143 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36917, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T15:29:15,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42689 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-26T15:29:15,162 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, size=6.7 K (6845bytes) 2024-11-26T15:29:15,162 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta 2024-11-26T15:29:15,162 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta after 0ms 2024-11-26T15:29:15,165 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta took 3ms 2024-11-26T15:29:15,171 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-26T15:29:15,172 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta so closing down 2024-11-26T15:29:15,172 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:15,174 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta.temp 2024-11-26T15:29:15,175 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta.temp 2024-11-26T15:29:15,175 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741892_1070 (size=6604) 2024-11-26T15:29:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741892_1070 (size=6604) 2024-11-26T15:29:15,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741892_1070 (size=6604) 2024-11-26T15:29:15,186 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:15,187 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta.temp to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T15:29:15,187 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 22 ms; skipped=1; WAL=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-26T15:29:15,188 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, journal: Splitting hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, size=6.7 K (6845bytes) at 1732634955162Finishing writing output for hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta so closing down at 1732634955172 (+10 ms)Creating recovered edits writer path=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta.temp at 1732634955175 (+3 ms)3 split writer threads finished at 1732634955176 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732634955186 (+10 ms)Rename recovered edits hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta.temp to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732634955187 (+1 ms)Processed 16 edits across 1 Regions in 22 ms; skipped=1; WAL=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1732634955188 (+1 ms) 2024-11-26T15:29:15,188 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta 2024-11-26T15:29:15,189 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-26T15:29:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-26T15:29:15,196 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs 2024-11-26T15:29:15,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-26T15:29:15,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, worker=6cc2bc8bf733,42689,1732634938394 in 224 msec 2024-11-26T15:29:15,200 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:15,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T15:29:15,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 6cc2bc8bf733%2C43161%2C1732634938266.meta.1732634940351.meta, worker=6cc2bc8bf733,42689,1732634938394 in 236 msec 2024-11-26T15:29:15,204 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting dir is empty, no logs to split. 2024-11-26T15:29:15,204 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6cc2bc8bf733,43161,1732634938266 WAL count=0, meta=true 2024-11-26T15:29:15,204 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 6cc2bc8bf733,43161,1732634938266 WAL splitting is done? wals=0, meta=true 2024-11-26T15:29:15,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T15:29:15,207 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T15:29:15,208 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T15:29:15,260 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:29:15,263 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1] 2024-11-26T15:29:15,264 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6cc2bc8bf733:43161 this server is in the failed servers list 2024-11-26T15:29:15,265 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6cc2bc8bf733:43161 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:15,265 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:15,265 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 from cache 2024-11-26T15:29:15,358 DEBUG [6cc2bc8bf733:44737 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T15:29:15,359 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(204): Hosts are {6cc2bc8bf733=0} racks are {/default-rack=0} 2024-11-26T15:29:15,359 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T15:29:15,359 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T15:29:15,359 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T15:29:15,359 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T15:29:15,359 INFO [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T15:29:15,359 INFO [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T15:29:15,359 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T15:29:15,360 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:15,361 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6cc2bc8bf733,42689,1732634938394, state=OPENING 2024-11-26T15:29:15,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:15,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:15,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:15,410 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T15:29:15,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:15,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:15,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:15,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,42689,1732634938394}] 2024-11-26T15:29:15,569 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T15:29:15,570 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:15,570 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T15:29:15,573 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6cc2bc8bf733%2C42689%2C1732634938394.meta, suffix=.meta, logDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,42689,1732634938394, archiveDir=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs, maxLogs=32 2024-11-26T15:29:15,586 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,42689,1732634938394/6cc2bc8bf733%2C42689%2C1732634938394.meta.1732634955574.meta, exclude list is [], retry=0 2024-11-26T15:29:15,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:15,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:15,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:15,591 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,42689,1732634938394/6cc2bc8bf733%2C42689%2C1732634938394.meta.1732634955574.meta 2024-11-26T15:29:15,591 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:15,591 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:15,592 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T15:29:15,592 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T15:29:15,592 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T15:29:15,592 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T15:29:15,592 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:15,592 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T15:29:15,592 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T15:29:15,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T15:29:15,594 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T15:29:15,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:15,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:15,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T15:29:15,596 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T15:29:15,596 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:15,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:15,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T15:29:15,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T15:29:15,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:15,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:15,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T15:29:15,599 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T15:29:15,599 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:15,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T15:29:15,599 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T15:29:15,600 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740 2024-11-26T15:29:15,602 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740 2024-11-26T15:29:15,603 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T15:29:15,607 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T15:29:15,607 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-26T15:29:15,626 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/info/84225345af8c4a05b95ec824faee3ecd is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4./info:regioninfo/1732634954551/Put/seqid=0 2024-11-26T15:29:15,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741894_1072 (size=11177) 2024-11-26T15:29:15,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741894_1072 (size=11177) 2024-11-26T15:29:15,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741894_1072 (size=11177) 2024-11-26T15:29:15,634 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/info/84225345af8c4a05b95ec824faee3ecd 2024-11-26T15:29:15,653 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/ns/9078774b670a482199cbd920c6db9998 is 43, key is default/ns:d/1732634940501/Put/seqid=0 2024-11-26T15:29:15,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741895_1073 (size=5153) 2024-11-26T15:29:15,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741895_1073 (size=5153) 2024-11-26T15:29:15,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741895_1073 (size=5153) 2024-11-26T15:29:15,660 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/ns/9078774b670a482199cbd920c6db9998 2024-11-26T15:29:15,678 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/table/79b6645070b94814ad9cb619b28550b5 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732634952795/Put/seqid=0 2024-11-26T15:29:15,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741896_1074 (size=5431) 2024-11-26T15:29:15,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741896_1074 (size=5431) 2024-11-26T15:29:15,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741896_1074 (size=5431) 2024-11-26T15:29:15,685 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/table/79b6645070b94814ad9cb619b28550b5 2024-11-26T15:29:15,694 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/info/84225345af8c4a05b95ec824faee3ecd as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/info/84225345af8c4a05b95ec824faee3ecd 2024-11-26T15:29:15,701 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/info/84225345af8c4a05b95ec824faee3ecd, entries=36, sequenceid=18, filesize=10.9 K 2024-11-26T15:29:15,702 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/ns/9078774b670a482199cbd920c6db9998 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/ns/9078774b670a482199cbd920c6db9998 2024-11-26T15:29:15,708 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/ns/9078774b670a482199cbd920c6db9998, entries=2, sequenceid=18, filesize=5.0 K 2024-11-26T15:29:15,710 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/table/79b6645070b94814ad9cb619b28550b5 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/table/79b6645070b94814ad9cb619b28550b5 2024-11-26T15:29:15,716 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/table/79b6645070b94814ad9cb619b28550b5, entries=2, sequenceid=18, filesize=5.3 K 2024-11-26T15:29:15,716 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=18, compaction requested=false; wal=null 2024-11-26T15:29:15,717 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T15:29:15,719 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T15:29:15,719 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T15:29:15,719 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T15:29:15,721 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T15:29:15,723 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-26T15:29:15,724 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66890485, jitterRate=-0.0032541006803512573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T15:29:15,724 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T15:29:15,725 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732634955592Writing region info on filesystem at 1732634955593 (+1 ms)Initializing all the Stores at 1732634955593Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634955593Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634955594 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634955594Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732634955594Obtaining lock to block concurrent updates at 1732634955607 (+13 ms)Preparing flush snapshotting stores in 1588230740 at 1732634955607Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732634955608 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732634955608Flushing 1588230740/info: creating writer at 1732634955608Flushing 1588230740/info: appending metadata at 1732634955626 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732634955626Flushing 1588230740/ns: creating writer at 1732634955640 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732634955652 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1732634955653 (+1 ms)Flushing 1588230740/table: creating writer at 1732634955665 (+12 ms)Flushing 1588230740/table: appending metadata at 1732634955678 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732634955678Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f1cf162: reopening flushed file at 1732634955693 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@415c43e2: reopening flushed file at 1732634955701 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cd401e7: reopening flushed file at 1732634955709 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=18, compaction requested=false; wal=null at 1732634955716 (+7 ms)Cleaning up temporary data from old regions at 1732634955719 (+3 ms)Running coprocessor post-open hooks at 1732634955724 (+5 ms)Region opened successfully at 1732634955725 (+1 ms) 2024-11-26T15:29:15,726 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732634955564 2024-11-26T15:29:15,729 DEBUG [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T15:29:15,729 INFO [RS_OPEN_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T15:29:15,729 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:15,730 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6cc2bc8bf733,42689,1732634938394, state=OPEN 2024-11-26T15:29:15,769 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:29:15,771 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,42689,1732634938394, seqNum=-1] 2024-11-26T15:29:15,771 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:29:15,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:29:15,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:15,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:15,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T15:29:15,778 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:15,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:15,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:15,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T15:29:15,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-26T15:29:15,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6cc2bc8bf733,42689,1732634938394 in 367 msec 2024-11-26T15:29:15,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-26T15:29:15,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 577 msec 2024-11-26T15:29:15,786 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 6cc2bc8bf733,43161,1732634938266 had 2 regions 2024-11-26T15:29:15,787 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6cc2bc8bf733,43161,1732634938266, splitWal=true, meta=true, isMeta: false 2024-11-26T15:29:15,789 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6cc2bc8bf733,43161,1732634938266 WAL count=1, meta=false 2024-11-26T15:29:15,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6cc2bc8bf733%2C43161%2C1732634938266.1732634939885}] 2024-11-26T15:29:15,791 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:15,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, worker=6cc2bc8bf733,42689,1732634938394}] 2024-11-26T15:29:15,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6cc2bc8bf733:43161 this server is in the failed servers list 2024-11-26T15:29:15,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6cc2bc8bf733:43161 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:15,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:15,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=12 from cache 2024-11-26T15:29:15,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42689 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-26T15:29:15,967 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, size=2.1 K (2142bytes) 2024-11-26T15:29:15,967 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 2024-11-26T15:29:15,967 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 after 0ms 2024-11-26T15:29:15,969 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 took 3ms 2024-11-26T15:29:15,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 1122cfb9a9b693d960440938ac541ca4: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-26T15:29:15,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 so closing down 2024-11-26T15:29:15,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:15,972 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:15,972 INFO [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-26T15:29:15,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, journal: Splitting hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, size=2.1 K (2142bytes) at 1732634955967Finishing writing output for hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 so closing down at 1732634955972 (+5 ms)3 split writer threads finished at 1732634955972Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732634955972 2024-11-26T15:29:15,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 2024-11-26T15:29:15,972 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6cc2bc8bf733:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-26T15:29:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44737 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-26T15:29:15,976 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting/6cc2bc8bf733%2C43161%2C1732634938266.1732634939885 to hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs 2024-11-26T15:29:15,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-26T15:29:15,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, worker=6cc2bc8bf733,42689,1732634938394 in 184 msec 2024-11-26T15:29:15,981 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:15,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-26T15:29:15,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 6cc2bc8bf733%2C43161%2C1732634938266.1732634939885, worker=6cc2bc8bf733,42689,1732634938394 in 192 msec 2024-11-26T15:29:15,985 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/WALs/6cc2bc8bf733,43161,1732634938266-splitting dir is empty, no logs to split. 2024-11-26T15:29:15,985 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6cc2bc8bf733,43161,1732634938266 WAL count=0, meta=false 2024-11-26T15:29:15,985 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6cc2bc8bf733,43161,1732634938266 WAL splitting is done? wals=0, meta=false 2024-11-26T15:29:15,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN}] 2024-11-26T15:29:15,989 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN 2024-11-26T15:29:15,990 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T15:29:16,109 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18] 2024-11-26T15:29:16,109 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6cc2bc8bf733:43161 this server is in the failed servers list 2024-11-26T15:29:16,110 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6cc2bc8bf733:43161 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:16,110 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:16,110 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18 from cache 2024-11-26T15:29:16,140 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(204): Hosts are {6cc2bc8bf733=0} racks are {/default-rack=0} 2024-11-26T15:29:16,140 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T15:29:16,140 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T15:29:16,141 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T15:29:16,141 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T15:29:16,141 INFO [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T15:29:16,141 INFO [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T15:29:16,141 DEBUG [6cc2bc8bf733:44737 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T15:29:16,141 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPENING, regionLocation=6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:16,142 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6cc2bc8bf733:43161 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6cc2bc8bf733/172.17.0.3:43161 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:16,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1, error=java.net.ConnectException: Call to address=6cc2bc8bf733:43161 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6cc2bc8bf733/172.17.0.3:43161 2024-11-26T15:29:16,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-26T15:29:16,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,43161,1732634938266, seqNum=-1 from cache 2024-11-26T15:29:16,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6cc2bc8bf733:43161 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6cc2bc8bf733/172.17.0.3:43161 2024-11-26T15:29:16,250 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T15:29:16,250 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6cc2bc8bf733,42689,1732634938394, seqNum=-1] 2024-11-26T15:29:16,250 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T15:29:16,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33377, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T15:29:16,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN because future has completed 2024-11-26T15:29:16,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354}] 2024-11-26T15:29:16,410 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:16,410 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 1122cfb9a9b693d960440938ac541ca4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:16,411 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,411 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:16,411 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,411 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,412 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,414 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf1 2024-11-26T15:29:16,414 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:16,420 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf1/785b5f9d8e714a39a99954502d7e3003 2024-11-26T15:29:16,420 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:16,421 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,422 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1122cfb9a9b693d960440938ac541ca4 columnFamilyName cf2 2024-11-26T15:29:16,422 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:16,426 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18] 2024-11-26T15:29:16,427 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6cc2bc8bf733:43161 this server is in the failed servers list 2024-11-26T15:29:16,427 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6cc2bc8bf733:43161 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:16,427 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6cc2bc8bf733:43161 2024-11-26T15:29:16,427 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,43161,1732634938266, seqNum=18 from cache 2024-11-26T15:29:16,429 DEBUG [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/cf2/890e6a5d7e9444d38470806337b23970 2024-11-26T15:29:16,429 INFO [StoreOpener-1122cfb9a9b693d960440938ac541ca4-1 {}] regionserver.HStore(327): Store=1122cfb9a9b693d960440938ac541ca4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:16,430 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,431 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,432 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,433 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,433 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,433 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T15:29:16,435 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,436 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 1122cfb9a9b693d960440938ac541ca4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70332990, jitterRate=0.04804322123527527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T15:29:16,436 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:16,437 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 1122cfb9a9b693d960440938ac541ca4: Running coprocessor pre-open hook at 1732634956411Writing region info on filesystem at 1732634956411Initializing all the Stores at 1732634956412 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634956412Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634956412Cleaning up temporary data from old regions at 1732634956433 (+21 ms)Running coprocessor post-open hooks at 1732634956436 (+3 ms)Region opened successfully at 1732634956437 (+1 ms) 2024-11-26T15:29:16,438 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., pid=21, masterSystemTime=1732634956406 2024-11-26T15:29:16,441 DEBUG [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:16,441 INFO [RS_OPEN_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:16,442 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1122cfb9a9b693d960440938ac541ca4, regionState=OPEN, openSeqNum=18, regionLocation=6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:16,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354 because future has completed 2024-11-26T15:29:16,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-26T15:29:16,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 1122cfb9a9b693d960440938ac541ca4, server=6cc2bc8bf733,41249,1732634938354 in 191 msec 2024-11-26T15:29:16,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-26T15:29:16,451 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 6cc2bc8bf733,43161,1732634938266 after splitting done 2024-11-26T15:29:16,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1122cfb9a9b693d960440938ac541ca4, ASSIGN in 461 msec 2024-11-26T15:29:16,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 6cc2bc8bf733,43161,1732634938266, splitWal=true, meta=true in 1.5030 sec 2024-11-26T15:29:16,948 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4., hostname=6cc2bc8bf733,41249,1732634938354, seqNum=18] 2024-11-26T15:29:16,962 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 411), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=9363 (was 9426) 2024-11-26T15:29:16,963 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-26T15:29:16,973 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=9362 2024-11-26T15:29:16,973 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-26T15:29:16,987 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:16,989 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:16,989 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:16,991 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-56025996, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-56025996, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:17,003 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-56025996/hregion-56025996.1732634956992, exclude list is [], retry=0 2024-11-26T15:29:17,006 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:17,006 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:17,006 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:17,008 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-56025996/hregion-56025996.1732634956992 2024-11-26T15:29:17,008 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:17,008 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 95dd52614ad9ab68654662663cceaf58, NAME => 'testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:17,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741898_1076 (size=67) 2024-11-26T15:29:17,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741898_1076 (size=67) 2024-11-26T15:29:17,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741898_1076 (size=67) 2024-11-26T15:29:17,017 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:17,019 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,020 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName a 2024-11-26T15:29:17,020 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,021 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,021 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,022 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName b 2024-11-26T15:29:17,023 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,023 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,023 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,024 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName c 2024-11-26T15:29:17,024 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,025 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,025 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,027 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,027 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,028 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:17,029 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,031 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:17,032 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 95dd52614ad9ab68654662663cceaf58; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70182542, jitterRate=0.045801371335983276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:17,033 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 95dd52614ad9ab68654662663cceaf58: Writing region info on filesystem at 1732634957018Initializing all the Stores at 1732634957018Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957018Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957019 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957019Cleaning up temporary data from old regions at 1732634957027 (+8 ms)Region opened successfully at 1732634957032 (+5 ms) 2024-11-26T15:29:17,033 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 95dd52614ad9ab68654662663cceaf58, disabling compactions & flushes 2024-11-26T15:29:17,033 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,033 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,033 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. after waiting 0 ms 2024-11-26T15:29:17,033 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,033 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,033 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 95dd52614ad9ab68654662663cceaf58: Waiting for close lock at 1732634957033Disabling compacts and flushes for region at 1732634957033Disabling writes for close at 1732634957033Writing region close event to WAL at 1732634957033Closed at 1732634957033 2024-11-26T15:29:17,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741897_1075 (size=93) 2024-11-26T15:29:17,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741897_1075 (size=93) 2024-11-26T15:29:17,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741897_1075 (size=93) 2024-11-26T15:29:17,038 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:17,038 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-56025996:(num 1732634956992) 2024-11-26T15:29:17,038 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:17,040 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:17,053 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040, exclude list is [], retry=0 2024-11-26T15:29:17,055 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:17,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:17,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:17,057 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 2024-11-26T15:29:17,058 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:17,058 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 95dd52614ad9ab68654662663cceaf58, NAME => 'testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:17,058 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:17,058 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,058 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,059 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,060 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName a 2024-11-26T15:29:17,060 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,061 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,061 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,061 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName b 2024-11-26T15:29:17,062 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,062 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,062 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,063 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName c 2024-11-26T15:29:17,063 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,063 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,063 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,065 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,066 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,066 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,067 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:17,068 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,068 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 95dd52614ad9ab68654662663cceaf58; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60392983, jitterRate=-0.10007442533969879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:17,069 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 95dd52614ad9ab68654662663cceaf58: Writing region info on filesystem at 1732634957058Initializing all the Stores at 1732634957059 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957059Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957059Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957059Cleaning up temporary data from old regions at 1732634957066 (+7 ms)Region opened successfully at 1732634957069 (+3 ms) 2024-11-26T15:29:17,095 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 95dd52614ad9ab68654662663cceaf58 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T15:29:17,115 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/a/18d8f95ce3b34e6780ae1032215280a8 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732634957069/Put/seqid=0 2024-11-26T15:29:17,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741900_1078 (size=5958) 2024-11-26T15:29:17,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741900_1078 (size=5958) 2024-11-26T15:29:17,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741900_1078 (size=5958) 2024-11-26T15:29:17,122 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/a/18d8f95ce3b34e6780ae1032215280a8 2024-11-26T15:29:17,142 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/b/f058599f6aae4adb8b5e697057ff6271 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732634957077/Put/seqid=0 2024-11-26T15:29:17,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741901_1079 (size=5958) 2024-11-26T15:29:17,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741901_1079 (size=5958) 2024-11-26T15:29:17,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741901_1079 (size=5958) 2024-11-26T15:29:17,550 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/b/f058599f6aae4adb8b5e697057ff6271 2024-11-26T15:29:17,578 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/c/98ad088282414021b1cf3c5aa6bf425a is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732634957085/Put/seqid=0 2024-11-26T15:29:17,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741902_1080 (size=5958) 2024-11-26T15:29:17,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741902_1080 (size=5958) 2024-11-26T15:29:17,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741902_1080 (size=5958) 2024-11-26T15:29:17,587 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/c/98ad088282414021b1cf3c5aa6bf425a 2024-11-26T15:29:17,594 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/a/18d8f95ce3b34e6780ae1032215280a8 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/a/18d8f95ce3b34e6780ae1032215280a8 2024-11-26T15:29:17,600 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/a/18d8f95ce3b34e6780ae1032215280a8, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T15:29:17,601 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/b/f058599f6aae4adb8b5e697057ff6271 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/b/f058599f6aae4adb8b5e697057ff6271 2024-11-26T15:29:17,606 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/b/f058599f6aae4adb8b5e697057ff6271, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T15:29:17,607 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/c/98ad088282414021b1cf3c5aa6bf425a as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/c/98ad088282414021b1cf3c5aa6bf425a 2024-11-26T15:29:17,613 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/c/98ad088282414021b1cf3c5aa6bf425a, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T15:29:17,615 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 95dd52614ad9ab68654662663cceaf58 in 520ms, sequenceid=33, compaction requested=false 2024-11-26T15:29:17,615 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 95dd52614ad9ab68654662663cceaf58: 2024-11-26T15:29:17,615 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 95dd52614ad9ab68654662663cceaf58, disabling compactions & flushes 2024-11-26T15:29:17,615 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,615 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,615 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. after waiting 0 ms 2024-11-26T15:29:17,615 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,616 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. 2024-11-26T15:29:17,617 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 95dd52614ad9ab68654662663cceaf58: Waiting for close lock at 1732634957615Disabling compacts and flushes for region at 1732634957615Disabling writes for close at 1732634957615Writing region close event to WAL at 1732634957616 (+1 ms)Closed at 1732634957616 2024-11-26T15:29:17,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741899_1077 (size=5937) 2024-11-26T15:29:17,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741899_1077 (size=5937) 2024-11-26T15:29:17,620 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 not finished, retry = 0 2024-11-26T15:29:17,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741899_1077 (size=5937) 2024-11-26T15:29:17,725 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/b/f058599f6aae4adb8b5e697057ff6271 to hdfs://localhost:37937/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/b/f058599f6aae4adb8b5e697057ff6271 2024-11-26T15:29:17,741 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040, size=5.8 K (5937bytes) 2024-11-26T15:29:17,741 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 2024-11-26T15:29:17,742 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 after 1ms 2024-11-26T15:29:17,744 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 took 3ms 2024-11-26T15:29:17,746 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 so closing down 2024-11-26T15:29:17,746 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:17,747 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732634957040.temp 2024-11-26T15:29:17,748 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000003-wal.1732634957040.temp 2024-11-26T15:29:17,748 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:17,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741903_1081 (size=5223) 2024-11-26T15:29:17,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741903_1081 (size=5223) 2024-11-26T15:29:17,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741903_1081 (size=5223) 2024-11-26T15:29:17,760 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000003-wal.1732634957040.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:17,761 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000003-wal.1732634957040.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000032 2024-11-26T15:29:17,761 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-26T15:29:17,761 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040, journal: Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040, size=5.8 K (5937bytes) at 1732634957741Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 so closing down at 1732634957746 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000003-wal.1732634957040.temp at 1732634957748 (+2 ms)3 split writer threads finished at 1732634957748Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000003-wal.1732634957040.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732634957760 (+12 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000003-wal.1732634957040.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000032 at 1732634957761 (+1 ms)Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732634957761 2024-11-26T15:29:17,763 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957040 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634957040 2024-11-26T15:29:17,764 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000032 2024-11-26T15:29:17,764 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:17,766 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:17,784 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957767, exclude list is [], retry=0 2024-11-26T15:29:17,787 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:17,787 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:17,787 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:17,795 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957767 2024-11-26T15:29:17,795 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:17,795 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 95dd52614ad9ab68654662663cceaf58, NAME => 'testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:17,795 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:17,795 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,795 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,797 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,798 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName a 2024-11-26T15:29:17,798 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,805 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/a/18d8f95ce3b34e6780ae1032215280a8 2024-11-26T15:29:17,805 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,805 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,806 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName b 2024-11-26T15:29:17,807 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,807 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,807 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,808 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95dd52614ad9ab68654662663cceaf58 columnFamilyName c 2024-11-26T15:29:17,808 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,814 DEBUG [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/c/98ad088282414021b1cf3c5aa6bf425a 2024-11-26T15:29:17,815 INFO [StoreOpener-95dd52614ad9ab68654662663cceaf58-1 {}] regionserver.HStore(327): Store=95dd52614ad9ab68654662663cceaf58/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,815 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,815 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,817 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000032 2024-11-26T15:29:17,820 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000032 2024-11-26T15:29:17,821 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 95dd52614ad9ab68654662663cceaf58 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T15:29:17,834 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/b/350fa2bf731746fea0971aeae4dfdcb9 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732634957077/Put/seqid=0 2024-11-26T15:29:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741905_1083 (size=5958) 2024-11-26T15:29:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741905_1083 (size=5958) 2024-11-26T15:29:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741905_1083 (size=5958) 2024-11-26T15:29:17,842 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/b/350fa2bf731746fea0971aeae4dfdcb9 2024-11-26T15:29:17,849 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/.tmp/b/350fa2bf731746fea0971aeae4dfdcb9 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/b/350fa2bf731746fea0971aeae4dfdcb9 2024-11-26T15:29:17,855 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/b/350fa2bf731746fea0971aeae4dfdcb9, entries=10, sequenceid=32, filesize=5.8 K 2024-11-26T15:29:17,855 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 95dd52614ad9ab68654662663cceaf58 in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-11-26T15:29:17,856 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/0000000000000000032 2024-11-26T15:29:17,857 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,857 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,858 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:17,859 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 95dd52614ad9ab68654662663cceaf58 2024-11-26T15:29:17,862 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/95dd52614ad9ab68654662663cceaf58/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-26T15:29:17,863 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 95dd52614ad9ab68654662663cceaf58; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74480465, jitterRate=0.10984541475772858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:17,864 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 95dd52614ad9ab68654662663cceaf58: Writing region info on filesystem at 1732634957795Initializing all the Stores at 1732634957796 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957796Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957797 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957797Obtaining lock to block concurrent updates at 1732634957821 (+24 ms)Preparing flush snapshotting stores in 95dd52614ad9ab68654662663cceaf58 at 1732634957821Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732634957821Flushing stores of testReplayEditsWrittenViaHRegion,,1732634956987.95dd52614ad9ab68654662663cceaf58. at 1732634957821Flushing 95dd52614ad9ab68654662663cceaf58/b: creating writer at 1732634957821Flushing 95dd52614ad9ab68654662663cceaf58/b: appending metadata at 1732634957834 (+13 ms)Flushing 95dd52614ad9ab68654662663cceaf58/b: closing flushed file at 1732634957834Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39d64676: reopening flushed file at 1732634957848 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 95dd52614ad9ab68654662663cceaf58 in 35ms, sequenceid=32, compaction requested=false; wal=null at 1732634957855 (+7 ms)Cleaning up temporary data from old regions at 1732634957857 (+2 ms)Region opened successfully at 1732634957864 (+7 ms) 2024-11-26T15:29:17,887 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=415 (was 405) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:37616 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:45798 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:60376 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:37392 [Waiting for operation #51] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:45978 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:60576 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1118 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 215), ProcessCount=11 (was 11), AvailableMemoryMB=9355 (was 9362) 2024-11-26T15:29:17,887 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-26T15:29:17,898 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1118, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=9354 2024-11-26T15:29:17,898 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-26T15:29:17,913 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:17,915 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:17,915 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:17,918 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57225826, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-57225826, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:17,930 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57225826/hregion-57225826.1732634957918, exclude list is [], retry=0 2024-11-26T15:29:17,933 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:17,933 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:17,933 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:17,935 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57225826/hregion-57225826.1732634957918 2024-11-26T15:29:17,935 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:17,935 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b6e3d507b535ec22476c2135996c9f0b, NAME => 'testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:17,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741907_1085 (size=68) 2024-11-26T15:29:17,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741907_1085 (size=68) 2024-11-26T15:29:17,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741907_1085 (size=68) 2024-11-26T15:29:17,944 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:17,946 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,948 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName a 2024-11-26T15:29:17,948 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,948 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,949 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,950 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName b 2024-11-26T15:29:17,950 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,951 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,951 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,952 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName c 2024-11-26T15:29:17,952 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:17,953 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:17,953 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,954 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,954 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,955 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,955 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,956 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:17,957 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:17,959 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:17,959 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b6e3d507b535ec22476c2135996c9f0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60744271, jitterRate=-0.09483982622623444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:17,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b6e3d507b535ec22476c2135996c9f0b: Writing region info on filesystem at 1732634957944Initializing all the Stores at 1732634957945 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957945Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957946 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634957946Cleaning up temporary data from old regions at 1732634957955 (+9 ms)Region opened successfully at 1732634957960 (+5 ms) 2024-11-26T15:29:17,960 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b6e3d507b535ec22476c2135996c9f0b, disabling compactions & flushes 2024-11-26T15:29:17,960 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:17,960 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:17,960 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. after waiting 0 ms 2024-11-26T15:29:17,960 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:17,961 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:17,961 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b6e3d507b535ec22476c2135996c9f0b: Waiting for close lock at 1732634957960Disabling compacts and flushes for region at 1732634957960Disabling writes for close at 1732634957960Writing region close event to WAL at 1732634957961 (+1 ms)Closed at 1732634957961 2024-11-26T15:29:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741906_1084 (size=93) 2024-11-26T15:29:17,964 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-57225826/hregion-57225826.1732634957918 not finished, retry = 0 2024-11-26T15:29:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741906_1084 (size=93) 2024-11-26T15:29:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741906_1084 (size=93) 2024-11-26T15:29:18,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:18,067 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57225826:(num 1732634957918) 2024-11-26T15:29:18,067 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:18,069 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:18,082 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069, exclude list is [], retry=0 2024-11-26T15:29:18,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:18,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:18,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:18,087 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069 2024-11-26T15:29:18,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:18,142 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b6e3d507b535ec22476c2135996c9f0b, NAME => 'testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:18,145 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,145 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:18,145 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,145 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,147 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,148 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName a 2024-11-26T15:29:18,148 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:18,149 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:18,149 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,150 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName b 2024-11-26T15:29:18,150 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:18,151 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:18,151 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,152 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName c 2024-11-26T15:29:18,152 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:18,153 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:18,153 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,155 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,156 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,156 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,157 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:18,158 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,159 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b6e3d507b535ec22476c2135996c9f0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60292062, jitterRate=-0.10157826542854309}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:18,159 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:18,160 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b6e3d507b535ec22476c2135996c9f0b: Running coprocessor pre-open hook at 1732634958145Writing region info on filesystem at 1732634958145Initializing all the Stores at 1732634958146 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634958147 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634958147Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634958147Cleaning up temporary data from old regions at 1732634958156 (+9 ms)Running coprocessor post-open hooks at 1732634958159 (+3 ms)Region opened successfully at 1732634958160 (+1 ms) 2024-11-26T15:29:18,172 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b6e3d507b535ec22476c2135996c9f0b 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-26T15:29:18,174 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:18,307 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:29:18,339 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-26T15:29:18,339 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:18,340 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T15:29:18,340 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:18,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-26T15:29:18,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:18,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-26T15:29:18,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:18,342 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T15:29:19,175 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:20,176 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:21,178 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:22,180 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:23,182 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:23,648 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:29:24,183 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:25,185 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:26,186 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:26,670 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T15:29:27,188 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b6e3d507b535ec22476c2135996c9f0b/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:27,189 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b6e3d507b535ec22476c2135996c9f0b: 2024-11-26T15:29:27,189 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:27,206 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b6e3d507b535ec22476c2135996c9f0b: 2024-11-26T15:29:27,206 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-26T15:29:27,206 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b6e3d507b535ec22476c2135996c9f0b, disabling compactions & flushes 2024-11-26T15:29:27,206 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:27,206 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:27,206 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. after waiting 0 ms 2024-11-26T15:29:27,206 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:27,207 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:27,207 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. 2024-11-26T15:29:27,207 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b6e3d507b535ec22476c2135996c9f0b: Waiting for close lock at 1732634967206Running coprocessor pre-close hooks at 1732634967206Disabling compacts and flushes for region at 1732634967206Disabling writes for close at 1732634967206Writing region close event to WAL at 1732634967207 (+1 ms)Running coprocessor post-close hooks at 1732634967207Closed at 1732634967207 2024-11-26T15:29:27,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741908_1086 (size=3941) 2024-11-26T15:29:27,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741908_1086 (size=3941) 2024-11-26T15:29:27,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741908_1086 (size=3941) 2024-11-26T15:29:27,225 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069, size=3.8 K (3941bytes) 2024-11-26T15:29:27,225 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069 2024-11-26T15:29:27,225 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069 after 0ms 2024-11-26T15:29:27,228 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069 took 3ms 2024-11-26T15:29:27,230 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069 so closing down 2024-11-26T15:29:27,230 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:27,231 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732634958069.temp 2024-11-26T15:29:27,232 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000004-wal.1732634958069.temp 2024-11-26T15:29:27,232 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741909_1087 (size=2983) 2024-11-26T15:29:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741909_1087 (size=2983) 2024-11-26T15:29:27,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741909_1087 (size=2983) 2024-11-26T15:29:27,238 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000004-wal.1732634958069.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:27,240 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000004-wal.1732634958069.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000026 2024-11-26T15:29:27,240 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-26T15:29:27,240 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069, journal: Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069, size=3.8 K (3941bytes) at 1732634967225Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069 so closing down at 1732634967230 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000004-wal.1732634958069.temp at 1732634967232 (+2 ms)3 split writer threads finished at 1732634967232Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000004-wal.1732634958069.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732634967238 (+6 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000004-wal.1732634958069.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000026 at 1732634967240 (+2 ms)Processed 23 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732634967240 2024-11-26T15:29:27,241 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634958069 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634958069 2024-11-26T15:29:27,242 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000026 2024-11-26T15:29:27,242 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:27,244 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:27,256 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634967244, exclude list is [], retry=0 2024-11-26T15:29:27,258 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:27,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:27,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:27,261 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634967244 2024-11-26T15:29:27,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:27,261 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b6e3d507b535ec22476c2135996c9f0b, NAME => 'testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:27,262 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,262 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:27,262 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,262 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,264 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,265 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName a 2024-11-26T15:29:27,265 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:27,265 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:27,266 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,266 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName b 2024-11-26T15:29:27,266 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:27,267 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:27,267 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,268 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6e3d507b535ec22476c2135996c9f0b columnFamilyName c 2024-11-26T15:29:27,268 DEBUG [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:27,268 INFO [StoreOpener-b6e3d507b535ec22476c2135996c9f0b-1 {}] regionserver.HStore(327): Store=b6e3d507b535ec22476c2135996c9f0b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:27,269 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,272 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000026 2024-11-26T15:29:27,328 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000026 2024-11-26T15:29:27,328 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b6e3d507b535ec22476c2135996c9f0b 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-26T15:29:27,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/a/510e7c81d54f46d597146c8b26a23324 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732634967196/Put/seqid=0 2024-11-26T15:29:27,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741911_1089 (size=5523) 2024-11-26T15:29:27,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741911_1089 (size=5523) 2024-11-26T15:29:27,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741911_1089 (size=5523) 2024-11-26T15:29:27,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/a/510e7c81d54f46d597146c8b26a23324 2024-11-26T15:29:27,370 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/b/53d6b90a287c41c398a4e87843951587 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732634967190/Put/seqid=0 2024-11-26T15:29:27,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741912_1090 (size=5524) 2024-11-26T15:29:27,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741912_1090 (size=5524) 2024-11-26T15:29:27,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741912_1090 (size=5524) 2024-11-26T15:29:27,378 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/b/53d6b90a287c41c398a4e87843951587 2024-11-26T15:29:27,397 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/c/6a254d13ec674d7eaae74535d11bab54 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732634967192/Put/seqid=0 2024-11-26T15:29:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741913_1091 (size=5457) 2024-11-26T15:29:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741913_1091 (size=5457) 2024-11-26T15:29:27,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741913_1091 (size=5457) 2024-11-26T15:29:27,406 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/c/6a254d13ec674d7eaae74535d11bab54 2024-11-26T15:29:27,411 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/a/510e7c81d54f46d597146c8b26a23324 as hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/a/510e7c81d54f46d597146c8b26a23324 2024-11-26T15:29:27,417 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/a/510e7c81d54f46d597146c8b26a23324, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T15:29:27,418 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/b/53d6b90a287c41c398a4e87843951587 as hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/b/53d6b90a287c41c398a4e87843951587 2024-11-26T15:29:27,423 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/b/53d6b90a287c41c398a4e87843951587, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T15:29:27,424 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/.tmp/c/6a254d13ec674d7eaae74535d11bab54 as hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/c/6a254d13ec674d7eaae74535d11bab54 2024-11-26T15:29:27,429 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/c/6a254d13ec674d7eaae74535d11bab54, entries=6, sequenceid=26, filesize=5.3 K 2024-11-26T15:29:27,429 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for b6e3d507b535ec22476c2135996c9f0b in 101ms, sequenceid=26, compaction requested=false; wal=null 2024-11-26T15:29:27,430 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/0000000000000000026 2024-11-26T15:29:27,431 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,431 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,432 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:27,433 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,436 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsAfterAbortingFlush/b6e3d507b535ec22476c2135996c9f0b/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-26T15:29:27,437 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b6e3d507b535ec22476c2135996c9f0b; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73645585, jitterRate=0.09740473330020905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:27,437 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6e3d507b535ec22476c2135996c9f0b 2024-11-26T15:29:27,437 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b6e3d507b535ec22476c2135996c9f0b: Running coprocessor pre-open hook at 1732634967262Writing region info on filesystem at 1732634967262Initializing all the Stores at 1732634967263 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634967263Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634967264 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634967264Obtaining lock to block concurrent updates at 1732634967328 (+64 ms)Preparing flush snapshotting stores in b6e3d507b535ec22476c2135996c9f0b at 1732634967328Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732634967329 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1732634957913.b6e3d507b535ec22476c2135996c9f0b. at 1732634967329Flushing b6e3d507b535ec22476c2135996c9f0b/a: creating writer at 1732634967329Flushing b6e3d507b535ec22476c2135996c9f0b/a: appending metadata at 1732634967343 (+14 ms)Flushing b6e3d507b535ec22476c2135996c9f0b/a: closing flushed file at 1732634967343Flushing b6e3d507b535ec22476c2135996c9f0b/b: creating writer at 1732634967356 (+13 ms)Flushing b6e3d507b535ec22476c2135996c9f0b/b: appending metadata at 1732634967369 (+13 ms)Flushing b6e3d507b535ec22476c2135996c9f0b/b: closing flushed file at 1732634967369Flushing b6e3d507b535ec22476c2135996c9f0b/c: creating writer at 1732634967383 (+14 ms)Flushing b6e3d507b535ec22476c2135996c9f0b/c: appending metadata at 1732634967397 (+14 ms)Flushing b6e3d507b535ec22476c2135996c9f0b/c: closing flushed file at 1732634967397Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b78492f: reopening flushed file at 1732634967410 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ee5975d: reopening flushed file at 1732634967417 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@618f4812: reopening flushed file at 1732634967423 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for b6e3d507b535ec22476c2135996c9f0b in 101ms, sequenceid=26, compaction requested=false; wal=null at 1732634967429 (+6 ms)Cleaning up temporary data from old regions at 1732634967431 (+2 ms)Running coprocessor post-open hooks at 1732634967437 (+6 ms)Region opened successfully at 1732634967437 2024-11-26T15:29:27,457 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 415), OpenFileDescriptor=1182 (was 1118) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=190 (was 215), ProcessCount=11 (was 11), AvailableMemoryMB=9351 (was 9354) 2024-11-26T15:29:27,457 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-26T15:29:27,468 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=190, ProcessCount=11, AvailableMemoryMB=9350 2024-11-26T15:29:27,468 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-26T15:29:27,484 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:27,486 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:27,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:27,490 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07426817, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-07426817, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:27,507 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07426817/hregion-07426817.1732634967490, exclude list is [], retry=0 2024-11-26T15:29:27,510 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:27,510 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:27,510 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:27,512 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07426817/hregion-07426817.1732634967490 2024-11-26T15:29:27,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:27,513 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e81ac858f86b6af83419f87efcbbf102, NAME => 'testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741915_1093 (size=61) 2024-11-26T15:29:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741915_1093 (size=61) 2024-11-26T15:29:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741915_1093 (size=61) 2024-11-26T15:29:27,525 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:27,526 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,527 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e81ac858f86b6af83419f87efcbbf102 columnFamilyName a 2024-11-26T15:29:27,527 DEBUG [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:27,528 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(327): Store=e81ac858f86b6af83419f87efcbbf102/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:27,528 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,528 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,528 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,529 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,529 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,530 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,532 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:27,532 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e81ac858f86b6af83419f87efcbbf102; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70538872, jitterRate=0.05111110210418701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:27,533 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e81ac858f86b6af83419f87efcbbf102: Writing region info on filesystem at 1732634967525Initializing all the Stores at 1732634967526 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634967526Cleaning up temporary data from old regions at 1732634967529 (+3 ms)Region opened successfully at 1732634967533 (+4 ms) 2024-11-26T15:29:27,533 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e81ac858f86b6af83419f87efcbbf102, disabling compactions & flushes 2024-11-26T15:29:27,533 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,533 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,533 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. after waiting 0 ms 2024-11-26T15:29:27,533 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,533 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,533 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e81ac858f86b6af83419f87efcbbf102: Waiting for close lock at 1732634967533Disabling compacts and flushes for region at 1732634967533Disabling writes for close at 1732634967533Writing region close event to WAL at 1732634967533Closed at 1732634967533 2024-11-26T15:29:27,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741914_1092 (size=93) 2024-11-26T15:29:27,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741914_1092 (size=93) 2024-11-26T15:29:27,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741914_1092 (size=93) 2024-11-26T15:29:27,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:27,538 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07426817:(num 1732634967490) 2024-11-26T15:29:27,538 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:27,540 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:27,552 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540, exclude list is [], retry=0 2024-11-26T15:29:27,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:27,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:27,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:27,557 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 2024-11-26T15:29:27,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:27,557 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e81ac858f86b6af83419f87efcbbf102, NAME => 'testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:27,557 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:27,557 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,557 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,559 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,560 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e81ac858f86b6af83419f87efcbbf102 columnFamilyName a 2024-11-26T15:29:27,560 DEBUG [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:27,561 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(327): Store=e81ac858f86b6af83419f87efcbbf102/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:27,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,561 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,563 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,563 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,565 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:27,565 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e81ac858f86b6af83419f87efcbbf102; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64794043, jitterRate=-0.034493520855903625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:27,566 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e81ac858f86b6af83419f87efcbbf102: Writing region info on filesystem at 1732634967558Initializing all the Stores at 1732634967558Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634967558Cleaning up temporary data from old regions at 1732634967563 (+5 ms)Region opened successfully at 1732634967566 (+3 ms) 2024-11-26T15:29:27,573 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e81ac858f86b6af83419f87efcbbf102, disabling compactions & flushes 2024-11-26T15:29:27,573 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,573 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,573 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. after waiting 0 ms 2024-11-26T15:29:27,573 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,573 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,573 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. 2024-11-26T15:29:27,574 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e81ac858f86b6af83419f87efcbbf102: Waiting for close lock at 1732634967573Disabling compacts and flushes for region at 1732634967573Disabling writes for close at 1732634967573Writing region close event to WAL at 1732634967573Closed at 1732634967573 2024-11-26T15:29:27,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741916_1094 (size=1623) 2024-11-26T15:29:27,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741916_1094 (size=1623) 2024-11-26T15:29:27,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741916_1094 (size=1623) 2024-11-26T15:29:27,577 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 not finished, retry = 0 2024-11-26T15:29:27,698 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540, size=1.6 K (1623bytes) 2024-11-26T15:29:27,698 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 2024-11-26T15:29:27,699 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 after 1ms 2024-11-26T15:29:27,701 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 took 3ms 2024-11-26T15:29:27,703 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 so closing down 2024-11-26T15:29:27,703 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:27,704 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732634967540.temp 2024-11-26T15:29:27,705 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000003-wal.1732634967540.temp 2024-11-26T15:29:27,706 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:27,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741917_1095 (size=1623) 2024-11-26T15:29:27,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741917_1095 (size=1623) 2024-11-26T15:29:27,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741917_1095 (size=1623) 2024-11-26T15:29:27,712 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000003-wal.1732634967540.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:27,713 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000003-wal.1732634967540.temp to hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000012 2024-11-26T15:29:27,713 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-26T15:29:27,714 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540, journal: Splitting hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540, size=1.6 K (1623bytes) at 1732634967698Finishing writing output for hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 so closing down at 1732634967703 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000003-wal.1732634967540.temp at 1732634967705 (+2 ms)3 split writer threads finished at 1732634967706 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000003-wal.1732634967540.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732634967712 (+6 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000003-wal.1732634967540.temp to hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000012 at 1732634967713 (+1 ms)Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732634967713 2024-11-26T15:29:27,715 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634967540 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634967540 2024-11-26T15:29:27,716 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000012 2024-11-26T15:29:28,087 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:28,090 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:28,103 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634968090, exclude list is [], retry=0 2024-11-26T15:29:28,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:28,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:28,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:28,108 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634968090 2024-11-26T15:29:28,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:28,108 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e81ac858f86b6af83419f87efcbbf102, NAME => 'testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:28,108 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:28,108 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,108 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,111 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,113 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e81ac858f86b6af83419f87efcbbf102 columnFamilyName a 2024-11-26T15:29:28,113 DEBUG [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,114 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(327): Store=e81ac858f86b6af83419f87efcbbf102/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,114 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,115 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,116 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,117 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000012 2024-11-26T15:29:28,119 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000012 2024-11-26T15:29:28,120 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e81ac858f86b6af83419f87efcbbf102 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-26T15:29:28,141 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/.tmp/a/db6a9a37898241d4ac2d0fbead06d1a8 is 79, key is testDatalossWhenInputError/a:x0/1732634967566/Put/seqid=0 2024-11-26T15:29:28,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741919_1097 (size=5808) 2024-11-26T15:29:28,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741919_1097 (size=5808) 2024-11-26T15:29:28,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741919_1097 (size=5808) 2024-11-26T15:29:28,147 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/.tmp/a/db6a9a37898241d4ac2d0fbead06d1a8 2024-11-26T15:29:28,155 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/.tmp/a/db6a9a37898241d4ac2d0fbead06d1a8 as hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/a/db6a9a37898241d4ac2d0fbead06d1a8 2024-11-26T15:29:28,164 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/a/db6a9a37898241d4ac2d0fbead06d1a8, entries=10, sequenceid=12, filesize=5.7 K 2024-11-26T15:29:28,164 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e81ac858f86b6af83419f87efcbbf102 in 45ms, sequenceid=12, compaction requested=false; wal=null 2024-11-26T15:29:28,165 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/0000000000000000012 2024-11-26T15:29:28,165 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,165 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,168 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,170 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-26T15:29:28,171 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e81ac858f86b6af83419f87efcbbf102; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61848787, jitterRate=-0.0783812552690506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:28,171 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e81ac858f86b6af83419f87efcbbf102: Writing region info on filesystem at 1732634968108Initializing all the Stores at 1732634968111 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968111Obtaining lock to block concurrent updates at 1732634968120 (+9 ms)Preparing flush snapshotting stores in e81ac858f86b6af83419f87efcbbf102 at 1732634968120Finished memstore snapshotting testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732634968120Flushing stores of testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102. at 1732634968120Flushing e81ac858f86b6af83419f87efcbbf102/a: creating writer at 1732634968120Flushing e81ac858f86b6af83419f87efcbbf102/a: appending metadata at 1732634968140 (+20 ms)Flushing e81ac858f86b6af83419f87efcbbf102/a: closing flushed file at 1732634968140Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59f84176: reopening flushed file at 1732634968153 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e81ac858f86b6af83419f87efcbbf102 in 45ms, sequenceid=12, compaction requested=false; wal=null at 1732634968164 (+11 ms)Cleaning up temporary data from old regions at 1732634968165 (+1 ms)Region opened successfully at 1732634968171 (+6 ms) 2024-11-26T15:29:28,174 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e81ac858f86b6af83419f87efcbbf102, NAME => 'testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:28,174 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732634967484.e81ac858f86b6af83419f87efcbbf102.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:28,174 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,174 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,176 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,177 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e81ac858f86b6af83419f87efcbbf102 columnFamilyName a 2024-11-26T15:29:28,177 DEBUG [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,182 DEBUG [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/a/db6a9a37898241d4ac2d0fbead06d1a8 2024-11-26T15:29:28,182 INFO [StoreOpener-e81ac858f86b6af83419f87efcbbf102-1 {}] regionserver.HStore(327): Store=e81ac858f86b6af83419f87efcbbf102/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,182 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,183 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,184 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,185 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,185 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,186 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e81ac858f86b6af83419f87efcbbf102 2024-11-26T15:29:28,189 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testDatalossWhenInputError/e81ac858f86b6af83419f87efcbbf102/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-26T15:29:28,190 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e81ac858f86b6af83419f87efcbbf102; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72749447, jitterRate=0.0840512365102768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T15:29:28,190 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e81ac858f86b6af83419f87efcbbf102: Writing region info on filesystem at 1732634968175Initializing all the Stores at 1732634968175Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968175Cleaning up temporary data from old regions at 1732634968185 (+10 ms)Region opened successfully at 1732634968190 (+5 ms) 2024-11-26T15:29:28,206 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47132 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47216 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:48360 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:52746 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:52780 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814321108-172.17.0.3-1732634932847:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=190 (was 190), ProcessCount=11 (was 11), AvailableMemoryMB=9342 (was 9350) 2024-11-26T15:29:28,206 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-26T15:29:28,217 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=190, ProcessCount=11, AvailableMemoryMB=9342 2024-11-26T15:29:28,217 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-26T15:29:28,234 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:28,235 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:28,236 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:28,238 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25603379, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-25603379, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:28,249 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25603379/hregion-25603379.1732634968238, exclude list is [], retry=0 2024-11-26T15:29:28,252 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:28,252 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:28,252 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:28,255 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25603379/hregion-25603379.1732634968238 2024-11-26T15:29:28,256 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:28,256 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ce7ac97234ec945198ffff0b705a36c6, NAME => 'testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741921_1099 (size=63) 2024-11-26T15:29:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741921_1099 (size=63) 2024-11-26T15:29:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741921_1099 (size=63) 2024-11-26T15:29:28,265 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:28,266 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,267 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName a 2024-11-26T15:29:28,267 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,268 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,268 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,269 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName b 2024-11-26T15:29:28,269 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,270 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,270 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,271 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName c 2024-11-26T15:29:28,271 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,272 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,272 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,275 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,275 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,275 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:28,276 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,278 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:28,279 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ce7ac97234ec945198ffff0b705a36c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70606830, jitterRate=0.05212375521659851}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:28,279 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ce7ac97234ec945198ffff0b705a36c6: Writing region info on filesystem at 1732634968265Initializing all the Stores at 1732634968265Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968265Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968266 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968266Cleaning up temporary data from old regions at 1732634968275 (+9 ms)Region opened successfully at 1732634968279 (+4 ms) 2024-11-26T15:29:28,279 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ce7ac97234ec945198ffff0b705a36c6, disabling compactions & flushes 2024-11-26T15:29:28,279 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:28,279 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:28,279 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. after waiting 0 ms 2024-11-26T15:29:28,279 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:28,280 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:28,280 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ce7ac97234ec945198ffff0b705a36c6: Waiting for close lock at 1732634968279Disabling compacts and flushes for region at 1732634968279Disabling writes for close at 1732634968279Writing region close event to WAL at 1732634968280 (+1 ms)Closed at 1732634968280 2024-11-26T15:29:28,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741920_1098 (size=93) 2024-11-26T15:29:28,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741920_1098 (size=93) 2024-11-26T15:29:28,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741920_1098 (size=93) 2024-11-26T15:29:28,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:28,285 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25603379:(num 1732634968238) 2024-11-26T15:29:28,285 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:28,287 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:28,300 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287, exclude list is [], retry=0 2024-11-26T15:29:28,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:28,303 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:28,303 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:28,304 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 2024-11-26T15:29:28,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049)] 2024-11-26T15:29:28,307 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ce7ac97234ec945198ffff0b705a36c6, NAME => 'testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:28,307 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:28,307 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,307 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,308 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,309 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName a 2024-11-26T15:29:28,309 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,310 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,310 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,311 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName b 2024-11-26T15:29:28,311 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,311 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,311 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,312 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName c 2024-11-26T15:29:28,312 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:28,312 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:28,313 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,313 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,314 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,315 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,315 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,316 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:28,317 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:28,318 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ce7ac97234ec945198ffff0b705a36c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75361563, jitterRate=0.12297479808330536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:28,319 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ce7ac97234ec945198ffff0b705a36c6: Writing region info on filesystem at 1732634968307Initializing all the Stores at 1732634968308 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968308Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968308Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634968308Cleaning up temporary data from old regions at 1732634968315 (+7 ms)Region opened successfully at 1732634968319 (+4 ms) 2024-11-26T15:29:28,322 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732634968322/Put/seqid=0 2024-11-26T15:29:28,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741923_1101 (size=4875) 2024-11-26T15:29:28,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741923_1101 (size=4875) 2024-11-26T15:29:28,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741923_1101 (size=4875) 2024-11-26T15:29:28,737 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732634968736/Put/seqid=0 2024-11-26T15:29:28,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741924_1102 (size=4875) 2024-11-26T15:29:28,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741924_1102 (size=4875) 2024-11-26T15:29:28,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741924_1102 (size=4875) 2024-11-26T15:29:28,748 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732634968748/Put/seqid=0 2024-11-26T15:29:28,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741925_1103 (size=4875) 2024-11-26T15:29:28,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741925_1103 (size=4875) 2024-11-26T15:29:28,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741925_1103 (size=4875) 2024-11-26T15:29:28,757 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in ce7ac97234ec945198ffff0b705a36c6/a 2024-11-26T15:29:28,762 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-26T15:29:28,762 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T15:29:28,763 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in ce7ac97234ec945198ffff0b705a36c6/a 2024-11-26T15:29:28,767 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-26T15:29:28,767 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T15:29:28,767 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in ce7ac97234ec945198ffff0b705a36c6/a 2024-11-26T15:29:28,770 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-26T15:29:28,771 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T15:29:28,771 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ce7ac97234ec945198ffff0b705a36c6 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-26T15:29:28,787 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/.tmp/a/8b46a48559ec4747b3b64d9d03e6264c is 55, key is testCompactedBulkLoadedFiles/a:a/1732634968319/Put/seqid=0 2024-11-26T15:29:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741926_1104 (size=5107) 2024-11-26T15:29:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741926_1104 (size=5107) 2024-11-26T15:29:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741926_1104 (size=5107) 2024-11-26T15:29:28,794 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/.tmp/a/8b46a48559ec4747b3b64d9d03e6264c 2024-11-26T15:29:28,798 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/.tmp/a/8b46a48559ec4747b3b64d9d03e6264c as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c 2024-11-26T15:29:28,803 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c, entries=1, sequenceid=4, filesize=5.0 K 2024-11-26T15:29:28,804 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for ce7ac97234ec945198ffff0b705a36c6 in 33ms, sequenceid=4, compaction requested=false 2024-11-26T15:29:28,804 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ce7ac97234ec945198ffff0b705a36c6: 2024-11-26T15:29:28,806 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_ 2024-11-26T15:29:28,807 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_ 2024-11-26T15:29:28,808 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_ 2024-11-26T15:29:28,808 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile0 into ce7ac97234ec945198ffff0b705a36c6/a as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_ - updating store file list. 2024-11-26T15:29:28,813 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:28,813 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_ into ce7ac97234ec945198ffff0b705a36c6/a 2024-11-26T15:29:28,813 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile0 into ce7ac97234ec945198ffff0b705a36c6/a (new location: hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_) 2024-11-26T15:29:28,815 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile1 into ce7ac97234ec945198ffff0b705a36c6/a as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_ - updating store file list. 2024-11-26T15:29:28,819 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:28,819 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_ into ce7ac97234ec945198ffff0b705a36c6/a 2024-11-26T15:29:28,819 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile1 into ce7ac97234ec945198ffff0b705a36c6/a (new location: hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_) 2024-11-26T15:29:28,820 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile2 into ce7ac97234ec945198ffff0b705a36c6/a as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_ - updating store file list. 2024-11-26T15:29:28,825 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 94afab9cbf0041b4b897aab944da0bcb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:28,825 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_ into ce7ac97234ec945198ffff0b705a36c6/a 2024-11-26T15:29:28,825 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37937/hbase/testCompactedBulkLoadedFiles/hfile2 into ce7ac97234ec945198ffff0b705a36c6/a (new location: hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_) 2024-11-26T15:29:28,833 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T15:29:28,833 DEBUG [Time-limited test {}] regionserver.HStore(1541): ce7ac97234ec945198ffff0b705a36c6/a is initiating major compaction (all files) 2024-11-26T15:29:28,833 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ce7ac97234ec945198ffff0b705a36c6/a in testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:28,833 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_] into tmpdir=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/.tmp, totalSize=19.3 K 2024-11-26T15:29:28,834 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8b46a48559ec4747b3b64d9d03e6264c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732634968319 2024-11-26T15:29:28,834 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T15:29:28,834 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T15:29:28,835 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 94afab9cbf0041b4b897aab944da0bcb_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T15:29:28,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/.tmp/a/7e2085db2a1e4c46bf81a09e0c0736c9 is 55, key is testCompactedBulkLoadedFiles/a:a/1732634968319/Put/seqid=0 2024-11-26T15:29:28,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741927_1105 (size=6154) 2024-11-26T15:29:28,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741927_1105 (size=6154) 2024-11-26T15:29:28,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741927_1105 (size=6154) 2024-11-26T15:29:28,857 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/.tmp/a/7e2085db2a1e4c46bf81a09e0c0736c9 as hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/7e2085db2a1e4c46bf81a09e0c0736c9 2024-11-26T15:29:28,863 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in ce7ac97234ec945198ffff0b705a36c6/a of ce7ac97234ec945198ffff0b705a36c6 into 7e2085db2a1e4c46bf81a09e0c0736c9(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T15:29:28,863 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ce7ac97234ec945198ffff0b705a36c6: 2024-11-26T15:29:28,863 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T15:29:28,863 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T15:29:28,892 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287, size=0 (0bytes) 2024-11-26T15:29:28,892 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 might be still open, length is 0 2024-11-26T15:29:28,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 2024-11-26T15:29:28,892 WARN [IPC Server handler 4 on default port 37937 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-26T15:29:28,893 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 after 1ms 2024-11-26T15:29:29,626 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:47234 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34329:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47234 dst: /127.0.0.1:34329 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34329 remote=/127.0.0.1:47234]. Total timeout mills is 60000, 59237 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:29,628 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:48402 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:35535:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48402 dst: /127.0.0.1:35535 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:29,628 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:52802 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39747:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52802 dst: /127.0.0.1:39747 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:29,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741922_1106 (size=1659) 2024-11-26T15:29:29,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741922_1106 (size=1659) 2024-11-26T15:29:32,712 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T15:29:32,894 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 after 4002ms 2024-11-26T15:29:32,897 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 took 4005ms 2024-11-26T15:29:32,899 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287; continuing. 2024-11-26T15:29:32,899 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 so closing down 2024-11-26T15:29:32,899 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:32,900 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732634968287.temp 2024-11-26T15:29:32,902 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000003-wal.1732634968287.temp 2024-11-26T15:29:32,902 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:32,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741928_1107 (size=698) 2024-11-26T15:29:32,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741928_1107 (size=698) 2024-11-26T15:29:32,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741928_1107 (size=698) 2024-11-26T15:29:32,914 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000003-wal.1732634968287.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:32,916 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000003-wal.1732634968287.temp to hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000008 2024-11-26T15:29:32,916 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T15:29:32,916 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287, journal: Splitting hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287, size=0 (0bytes) at 1732634968892Finishing writing output for hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 so closing down at 1732634972899 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000003-wal.1732634968287.temp at 1732634972902 (+3 ms)3 split writer threads finished at 1732634972902Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000003-wal.1732634968287.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732634972914 (+12 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000003-wal.1732634968287.temp to hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000008 at 1732634972916 (+2 ms)Processed 5 edits across 1 Regions in 19 ms; skipped=3; WAL=hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287, size=0, length=0, corrupted=false, cancelled=false at 1732634972916 2024-11-26T15:29:32,917 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634968287 2024-11-26T15:29:32,918 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000008 2024-11-26T15:29:32,918 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:32,920 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:32,931 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634972920, exclude list is [], retry=0 2024-11-26T15:29:32,934 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:32,934 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:32,934 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:32,936 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634972920 2024-11-26T15:29:32,936 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:32,936 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ce7ac97234ec945198ffff0b705a36c6, NAME => 'testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:32,936 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:32,936 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,936 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,938 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,938 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName a 2024-11-26T15:29:32,939 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:32,945 DEBUG [StoreFileOpener-ce7ac97234ec945198ffff0b705a36c6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:32,945 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_ 2024-11-26T15:29:32,949 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/7e2085db2a1e4c46bf81a09e0c0736c9 2024-11-26T15:29:32,954 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c 2024-11-26T15:29:32,958 DEBUG [StoreFileOpener-ce7ac97234ec945198ffff0b705a36c6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 94afab9cbf0041b4b897aab944da0bcb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:32,958 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_ 2024-11-26T15:29:32,961 DEBUG [StoreFileOpener-ce7ac97234ec945198ffff0b705a36c6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T15:29:32,961 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_ 2024-11-26T15:29:32,961 WARN [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f83b12b 2024-11-26T15:29:32,961 WARN [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f83b12b 2024-11-26T15:29:32,962 WARN [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f83b12b 2024-11-26T15:29:32,962 WARN [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@1f83b12b 2024-11-26T15:29:32,962 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_] to archive 2024-11-26T15:29:32,962 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T15:29:32,964 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_ to hdfs://localhost:37937/hbase/archive/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_ 2024-11-26T15:29:32,965 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c to hdfs://localhost:37937/hbase/archive/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/8b46a48559ec4747b3b64d9d03e6264c 2024-11-26T15:29:32,966 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_ to hdfs://localhost:37937/hbase/archive/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/94afab9cbf0041b4b897aab944da0bcb_SeqId_4_ 2024-11-26T15:29:32,967 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_ to hdfs://localhost:37937/hbase/archive/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/a/ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_ 2024-11-26T15:29:32,967 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:32,967 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,968 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName b 2024-11-26T15:29:32,968 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:32,968 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:32,968 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,969 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce7ac97234ec945198ffff0b705a36c6 columnFamilyName c 2024-11-26T15:29:32,969 DEBUG [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:32,969 INFO [StoreOpener-ce7ac97234ec945198ffff0b705a36c6-1 {}] regionserver.HStore(327): Store=ce7ac97234ec945198ffff0b705a36c6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:32,969 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,970 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,971 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,972 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000008 2024-11-26T15:29:32,974 DEBUG [Time-limited test {}] regionserver.HRegion(5836): ce7ac97234ec945198ffff0b705a36c6 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "ce7ac97234ec945198ffff0b705a36c6" family_name: "a" compaction_input: "8b46a48559ec4747b3b64d9d03e6264c" compaction_input: "ec8dcb6fd4ba4e3e9129792118b7f1aa_SeqId_4_" compaction_input: "6e389e1a14dc4549847c38c00cd1e0cc_SeqId_4_" compaction_input: "94afab9cbf0041b4b897aab944da0bcb_SeqId_4_" compaction_output: "7e2085db2a1e4c46bf81a09e0c0736c9" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-26T15:29:32,974 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-26T15:29:32,974 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000008 2024-11-26T15:29:32,975 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/0000000000000000008 2024-11-26T15:29:32,976 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,976 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,977 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:32,978 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ce7ac97234ec945198ffff0b705a36c6 2024-11-26T15:29:32,980 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testCompactedBulkLoadedFiles/ce7ac97234ec945198ffff0b705a36c6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T15:29:32,981 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ce7ac97234ec945198ffff0b705a36c6; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65139727, jitterRate=-0.029342427849769592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:32,981 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ce7ac97234ec945198ffff0b705a36c6: Writing region info on filesystem at 1732634972936Initializing all the Stores at 1732634972937 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634972937Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634972938 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634972938Cleaning up temporary data from old regions at 1732634972976 (+38 ms)Region opened successfully at 1732634972981 (+5 ms) 2024-11-26T15:29:32,983 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ce7ac97234ec945198ffff0b705a36c6, disabling compactions & flushes 2024-11-26T15:29:32,983 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:32,983 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:32,983 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. after waiting 0 ms 2024-11-26T15:29:32,983 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:32,984 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732634968234.ce7ac97234ec945198ffff0b705a36c6. 2024-11-26T15:29:32,984 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ce7ac97234ec945198ffff0b705a36c6: Waiting for close lock at 1732634972983Disabling compacts and flushes for region at 1732634972983Disabling writes for close at 1732634972983Writing region close event to WAL at 1732634972984 (+1 ms)Closed at 1732634972984 2024-11-26T15:29:32,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741929_1108 (size=93) 2024-11-26T15:29:32,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741929_1108 (size=93) 2024-11-26T15:29:32,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741929_1108 (size=93) 2024-11-26T15:29:32,988 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:32,989 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732634972920) 2024-11-26T15:29:33,000 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=439 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-118156352_22 at /127.0.0.1:39660 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:37937 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:37937 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-118156352_22 at /127.0.0.1:54324 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-118156352_22 at /127.0.0.1:56548 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=175 (was 190), ProcessCount=11 (was 11), AvailableMemoryMB=9331 (was 9342) 2024-11-26T15:29:33,000 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-26T15:29:33,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=439, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=175, ProcessCount=11, AvailableMemoryMB=9330 2024-11-26T15:29:33,012 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-26T15:29:33,025 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:33,026 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T15:29:33,027 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T15:29:33,029 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-69641616, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/hregion-69641616, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:33,042 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-69641616/hregion-69641616.1732634973029, exclude list is [], retry=0 2024-11-26T15:29:33,044 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:33,044 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:33,045 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:33,047 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-69641616/hregion-69641616.1732634973029 2024-11-26T15:29:33,047 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:33,047 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d49177ea61caad30804146ea5c80f2ab, NAME => 'testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37937/hbase 2024-11-26T15:29:33,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741931_1110 (size=67) 2024-11-26T15:29:33,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741931_1110 (size=67) 2024-11-26T15:29:33,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741931_1110 (size=67) 2024-11-26T15:29:33,056 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:33,057 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,058 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName a 2024-11-26T15:29:33,058 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,059 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,059 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,060 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName b 2024-11-26T15:29:33,060 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,061 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,061 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,062 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName c 2024-11-26T15:29:33,062 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,062 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,063 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,063 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,063 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,064 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,064 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,065 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:33,066 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,068 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T15:29:33,068 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d49177ea61caad30804146ea5c80f2ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62118455, jitterRate=-0.07436288893222809}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:33,069 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d49177ea61caad30804146ea5c80f2ab: Writing region info on filesystem at 1732634973056Initializing all the Stores at 1732634973057 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973057Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973057Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973057Cleaning up temporary data from old regions at 1732634973064 (+7 ms)Region opened successfully at 1732634973069 (+5 ms) 2024-11-26T15:29:33,069 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d49177ea61caad30804146ea5c80f2ab, disabling compactions & flushes 2024-11-26T15:29:33,069 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,069 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,069 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. after waiting 0 ms 2024-11-26T15:29:33,069 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,069 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,069 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d49177ea61caad30804146ea5c80f2ab: Waiting for close lock at 1732634973069Disabling compacts and flushes for region at 1732634973069Disabling writes for close at 1732634973069Writing region close event to WAL at 1732634973069Closed at 1732634973069 2024-11-26T15:29:33,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741930_1109 (size=93) 2024-11-26T15:29:33,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741930_1109 (size=93) 2024-11-26T15:29:33,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741930_1109 (size=93) 2024-11-26T15:29:33,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:33,077 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-69641616:(num 1732634973029) 2024-11-26T15:29:33,077 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:33,078 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:33,091 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079, exclude list is [], retry=0 2024-11-26T15:29:33,093 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:33,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:33,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:33,095 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079 2024-11-26T15:29:33,095 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45859:45859),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:39591:39591)] 2024-11-26T15:29:33,095 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d49177ea61caad30804146ea5c80f2ab, NAME => 'testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:33,096 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:33,096 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,096 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,099 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,100 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName a 2024-11-26T15:29:33,100 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,100 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,100 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,101 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName b 2024-11-26T15:29:33,101 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,101 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,101 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,102 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName c 2024-11-26T15:29:33,102 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,102 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,102 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,103 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,104 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,105 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,105 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,105 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:33,106 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,107 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d49177ea61caad30804146ea5c80f2ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61338979, jitterRate=-0.08597798645496368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:33,107 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d49177ea61caad30804146ea5c80f2ab: Writing region info on filesystem at 1732634973096Initializing all the Stores at 1732634973097 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973097Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973099 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973099Cleaning up temporary data from old regions at 1732634973105 (+6 ms)Region opened successfully at 1732634973107 (+2 ms) 2024-11-26T15:29:33,113 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d49177ea61caad30804146ea5c80f2ab 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T15:29:33,128 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/a/ede9276337804c36bd56f84277e49c2c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732634973107/Put/seqid=0 2024-11-26T15:29:33,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741933_1112 (size=5958) 2024-11-26T15:29:33,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741933_1112 (size=5958) 2024-11-26T15:29:33,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741933_1112 (size=5958) 2024-11-26T15:29:33,135 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/a/ede9276337804c36bd56f84277e49c2c 2024-11-26T15:29:33,140 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/a/ede9276337804c36bd56f84277e49c2c as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/a/ede9276337804c36bd56f84277e49c2c 2024-11-26T15:29:33,145 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/a/ede9276337804c36bd56f84277e49c2c, entries=10, sequenceid=13, filesize=5.8 K 2024-11-26T15:29:33,146 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for d49177ea61caad30804146ea5c80f2ab in 33ms, sequenceid=13, compaction requested=false 2024-11-26T15:29:33,146 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d49177ea61caad30804146ea5c80f2ab: 2024-11-26T15:29:33,162 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d49177ea61caad30804146ea5c80f2ab, disabling compactions & flushes 2024-11-26T15:29:33,162 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,162 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,162 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. after waiting 0 ms 2024-11-26T15:29:33,162 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,163 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,163 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:33,163 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d49177ea61caad30804146ea5c80f2ab: Waiting for close lock at 1732634973162Disabling compacts and flushes for region at 1732634973162Disabling writes for close at 1732634973162Writing region close event to WAL at 1732634973163 (+1 ms)Closed at 1732634973163 2024-11-26T15:29:33,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741932_1111 (size=5869) 2024-11-26T15:29:33,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741932_1111 (size=5869) 2024-11-26T15:29:33,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741932_1111 (size=5869) 2024-11-26T15:29:33,183 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079, size=5.7 K (5869bytes) 2024-11-26T15:29:33,183 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079 2024-11-26T15:29:33,184 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079 after 1ms 2024-11-26T15:29:33,186 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079 took 3ms 2024-11-26T15:29:33,188 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079 so closing down 2024-11-26T15:29:33,188 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:33,190 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732634973079.temp 2024-11-26T15:29:33,191 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000003-wal.1732634973079.temp 2024-11-26T15:29:33,191 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:33,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741934_1113 (size=5223) 2024-11-26T15:29:33,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741934_1113 (size=5223) 2024-11-26T15:29:33,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741934_1113 (size=5223) 2024-11-26T15:29:33,201 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000003-wal.1732634973079.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:33,203 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000003-wal.1732634973079.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000035 2024-11-26T15:29:33,203 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-26T15:29:33,203 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079, journal: Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079, size=5.7 K (5869bytes) at 1732634973183Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079 so closing down at 1732634973188 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000003-wal.1732634973079.temp at 1732634973191 (+3 ms)3 split writer threads finished at 1732634973191Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000003-wal.1732634973079.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732634973201 (+10 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000003-wal.1732634973079.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000035 at 1732634973203 (+2 ms)Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732634973203 2024-11-26T15:29:33,205 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973079 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634973079 2024-11-26T15:29:33,206 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000035 2024-11-26T15:29:33,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:33,207 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:33,220 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208, exclude list is [], retry=0 2024-11-26T15:29:33,222 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:33,222 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:33,223 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:33,224 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 2024-11-26T15:29:33,224 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:33,225 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d49177ea61caad30804146ea5c80f2ab, NAME => 'testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab.', STARTKEY => '', ENDKEY => ''} 2024-11-26T15:29:33,225 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:33,225 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,225 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,226 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,227 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName a 2024-11-26T15:29:33,227 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,233 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/a/ede9276337804c36bd56f84277e49c2c 2024-11-26T15:29:33,233 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,233 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,234 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName b 2024-11-26T15:29:33,235 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,235 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,235 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,236 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName c 2024-11-26T15:29:33,236 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:33,236 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:33,236 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,239 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000035 2024-11-26T15:29:33,242 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000035 2024-11-26T15:29:33,242 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d49177ea61caad30804146ea5c80f2ab 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-26T15:29:33,257 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/b/6b081ae2765841238b8a295a3caeda3a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732634973146/Put/seqid=0 2024-11-26T15:29:33,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741936_1115 (size=5958) 2024-11-26T15:29:33,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741936_1115 (size=5958) 2024-11-26T15:29:33,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741936_1115 (size=5958) 2024-11-26T15:29:33,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/b/6b081ae2765841238b8a295a3caeda3a 2024-11-26T15:29:33,282 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/c/a2fa3eb0ff0b42a79756d59422896f74 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732634973153/Put/seqid=0 2024-11-26T15:29:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741937_1116 (size=5958) 2024-11-26T15:29:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741937_1116 (size=5958) 2024-11-26T15:29:33,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741937_1116 (size=5958) 2024-11-26T15:29:33,289 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/c/a2fa3eb0ff0b42a79756d59422896f74 2024-11-26T15:29:33,294 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/b/6b081ae2765841238b8a295a3caeda3a as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/b/6b081ae2765841238b8a295a3caeda3a 2024-11-26T15:29:33,299 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/b/6b081ae2765841238b8a295a3caeda3a, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T15:29:33,300 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/c/a2fa3eb0ff0b42a79756d59422896f74 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/c/a2fa3eb0ff0b42a79756d59422896f74 2024-11-26T15:29:33,305 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/c/a2fa3eb0ff0b42a79756d59422896f74, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T15:29:33,305 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d49177ea61caad30804146ea5c80f2ab in 63ms, sequenceid=35, compaction requested=false; wal=null 2024-11-26T15:29:33,306 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000035 2024-11-26T15:29:33,307 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,307 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,308 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:33,309 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:33,312 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-26T15:29:33,312 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d49177ea61caad30804146ea5c80f2ab; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75337553, jitterRate=0.12261702120304108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:33,313 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d49177ea61caad30804146ea5c80f2ab: Writing region info on filesystem at 1732634973225Initializing all the Stores at 1732634973226 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973226Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973226Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634973226Obtaining lock to block concurrent updates at 1732634973242 (+16 ms)Preparing flush snapshotting stores in d49177ea61caad30804146ea5c80f2ab at 1732634973242Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732634973242Flushing stores of testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. at 1732634973242Flushing d49177ea61caad30804146ea5c80f2ab/b: creating writer at 1732634973242Flushing d49177ea61caad30804146ea5c80f2ab/b: appending metadata at 1732634973256 (+14 ms)Flushing d49177ea61caad30804146ea5c80f2ab/b: closing flushed file at 1732634973256Flushing d49177ea61caad30804146ea5c80f2ab/c: creating writer at 1732634973268 (+12 ms)Flushing d49177ea61caad30804146ea5c80f2ab/c: appending metadata at 1732634973282 (+14 ms)Flushing d49177ea61caad30804146ea5c80f2ab/c: closing flushed file at 1732634973282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62939ae: reopening flushed file at 1732634973293 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@735f98ff: reopening flushed file at 1732634973299 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d49177ea61caad30804146ea5c80f2ab in 63ms, sequenceid=35, compaction requested=false; wal=null at 1732634973305 (+6 ms)Cleaning up temporary data from old regions at 1732634973307 (+2 ms)Region opened successfully at 1732634973312 (+5 ms) 2024-11-26T15:29:33,374 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208, size=0 (0bytes) 2024-11-26T15:29:33,374 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 might be still open, length is 0 2024-11-26T15:29:33,374 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 2024-11-26T15:29:33,374 WARN [IPC Server handler 1 on default port 37937 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-26T15:29:33,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 after 1ms 2024-11-26T15:29:35,623 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:39708 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39747:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39708 dst: /127.0.0.1:39747 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39747 remote=/127.0.0.1:39708]. Total timeout mills is 60000, 57714 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:35,624 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:54428 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:35535:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54428 dst: /127.0.0.1:35535 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:35,624 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1674615901_22 at /127.0.0.1:56590 [Receiving block BP-814321108-172.17.0.3-1732634932847:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34329:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56590 dst: /127.0.0.1:34329 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:35,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741935_1117 (size=5215) 2024-11-26T15:29:35,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741935_1117 (size=5215) 2024-11-26T15:29:35,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741935_1117 (size=5215) 2024-11-26T15:29:37,376 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 after 4002ms 2024-11-26T15:29:37,383 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 took 4009ms 2024-11-26T15:29:37,387 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208; continuing. 2024-11-26T15:29:37,387 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 so closing down 2024-11-26T15:29:37,387 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T15:29:37,389 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732634973208.temp 2024-11-26T15:29:37,390 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000037-wal.1732634973208.temp 2024-11-26T15:29:37,390 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T15:29:37,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741938_1118 (size=5223) 2024-11-26T15:29:37,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741938_1118 (size=5223) 2024-11-26T15:29:37,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741938_1118 (size=5223) 2024-11-26T15:29:37,396 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000037-wal.1732634973208.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T15:29:37,397 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000037-wal.1732634973208.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000066 2024-11-26T15:29:37,397 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T15:29:37,397 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208, journal: Splitting hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208, size=0 (0bytes) at 1732634973374Finishing writing output for hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 so closing down at 1732634977387 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000037-wal.1732634973208.temp at 1732634977390 (+3 ms)3 split writer threads finished at 1732634977390Closed recovered edits writer path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000037-wal.1732634973208.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732634977396 (+6 ms)Rename recovered edits hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000037-wal.1732634973208.temp to hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000066 at 1732634977397 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208, size=0, length=0, corrupted=false, cancelled=false at 1732634977397 2024-11-26T15:29:37,398 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 to hdfs://localhost:37937/hbase/oldWALs/wal.1732634973208 2024-11-26T15:29:37,399 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000066 2024-11-26T15:29:37,399 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T15:29:37,400 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37937/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024, archiveDir=hdfs://localhost:37937/hbase/oldWALs, maxLogs=32 2024-11-26T15:29:37,413 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634977401, exclude list is [], retry=0 2024-11-26T15:29:37,415 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39747,DS-aef6346b-d853-45b3-9671-8aa68d3dd674,DISK] 2024-11-26T15:29:37,415 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35535,DS-9bd479b4-dd44-4459-b9ee-ddc71945f6a3,DISK] 2024-11-26T15:29:37,415 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34329,DS-1545ab5a-a091-4f11-9908-0f7d1d4c2624,DISK] 2024-11-26T15:29:37,417 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634977401 2024-11-26T15:29:37,417 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39591:39591),(127.0.0.1/127.0.0.1:36049:36049),(127.0.0.1/127.0.0.1:45859:45859)] 2024-11-26T15:29:37,417 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T15:29:37,418 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,419 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName a 2024-11-26T15:29:37,419 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:37,424 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/a/ede9276337804c36bd56f84277e49c2c 2024-11-26T15:29:37,424 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:37,424 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,425 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName b 2024-11-26T15:29:37,425 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:37,429 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/b/6b081ae2765841238b8a295a3caeda3a 2024-11-26T15:29:37,430 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:37,430 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,431 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d49177ea61caad30804146ea5c80f2ab columnFamilyName c 2024-11-26T15:29:37,431 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T15:29:37,435 DEBUG [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/c/a2fa3eb0ff0b42a79756d59422896f74 2024-11-26T15:29:37,435 INFO [StoreOpener-d49177ea61caad30804146ea5c80f2ab-1 {}] regionserver.HStore(327): Store=d49177ea61caad30804146ea5c80f2ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T15:29:37,435 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,438 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,438 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000066 2024-11-26T15:29:37,443 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000066 2024-11-26T15:29:37,443 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d49177ea61caad30804146ea5c80f2ab 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T15:29:37,457 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/a/b7a486bddc3f4935aa2bcce7e8cd136c is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732634973319/Put/seqid=0 2024-11-26T15:29:37,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741940_1120 (size=5958) 2024-11-26T15:29:37,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741940_1120 (size=5958) 2024-11-26T15:29:37,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741940_1120 (size=5958) 2024-11-26T15:29:37,463 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/a/b7a486bddc3f4935aa2bcce7e8cd136c 2024-11-26T15:29:37,482 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/b/8209f776b2a748ddb597b76abffac0f8 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732634973326/Put/seqid=0 2024-11-26T15:29:37,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741941_1121 (size=5958) 2024-11-26T15:29:37,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741941_1121 (size=5958) 2024-11-26T15:29:37,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741941_1121 (size=5958) 2024-11-26T15:29:37,488 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/b/8209f776b2a748ddb597b76abffac0f8 2024-11-26T15:29:37,507 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/c/7fcfd1fb34e149ac8af1239f6ce7635c is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732634973331/Put/seqid=0 2024-11-26T15:29:37,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741942_1122 (size=5958) 2024-11-26T15:29:37,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741942_1122 (size=5958) 2024-11-26T15:29:37,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741942_1122 (size=5958) 2024-11-26T15:29:37,513 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/c/7fcfd1fb34e149ac8af1239f6ce7635c 2024-11-26T15:29:37,517 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/a/b7a486bddc3f4935aa2bcce7e8cd136c as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/a/b7a486bddc3f4935aa2bcce7e8cd136c 2024-11-26T15:29:37,522 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/a/b7a486bddc3f4935aa2bcce7e8cd136c, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T15:29:37,522 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/b/8209f776b2a748ddb597b76abffac0f8 as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/b/8209f776b2a748ddb597b76abffac0f8 2024-11-26T15:29:37,526 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/b/8209f776b2a748ddb597b76abffac0f8, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T15:29:37,527 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/.tmp/c/7fcfd1fb34e149ac8af1239f6ce7635c as hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/c/7fcfd1fb34e149ac8af1239f6ce7635c 2024-11-26T15:29:37,532 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/c/7fcfd1fb34e149ac8af1239f6ce7635c, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T15:29:37,532 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d49177ea61caad30804146ea5c80f2ab in 89ms, sequenceid=66, compaction requested=false; wal=null 2024-11-26T15:29:37,533 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/0000000000000000066 2024-11-26T15:29:37,534 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,534 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,534 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T15:29:37,536 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d49177ea61caad30804146ea5c80f2ab 2024-11-26T15:29:37,538 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/hbase/data/default/testReplayEditsWrittenViaHRegion/d49177ea61caad30804146ea5c80f2ab/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-26T15:29:37,538 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d49177ea61caad30804146ea5c80f2ab; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64403278, jitterRate=-0.040316373109817505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T15:29:37,539 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d49177ea61caad30804146ea5c80f2ab: Writing region info on filesystem at 1732634977417Initializing all the Stores at 1732634977418 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634977418Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634977418Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732634977418Obtaining lock to block concurrent updates at 1732634977443 (+25 ms)Preparing flush snapshotting stores in d49177ea61caad30804146ea5c80f2ab at 1732634977443Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732634977444 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. at 1732634977444Flushing d49177ea61caad30804146ea5c80f2ab/a: creating writer at 1732634977444Flushing d49177ea61caad30804146ea5c80f2ab/a: appending metadata at 1732634977456 (+12 ms)Flushing d49177ea61caad30804146ea5c80f2ab/a: closing flushed file at 1732634977456Flushing d49177ea61caad30804146ea5c80f2ab/b: creating writer at 1732634977467 (+11 ms)Flushing d49177ea61caad30804146ea5c80f2ab/b: appending metadata at 1732634977482 (+15 ms)Flushing d49177ea61caad30804146ea5c80f2ab/b: closing flushed file at 1732634977482Flushing d49177ea61caad30804146ea5c80f2ab/c: creating writer at 1732634977492 (+10 ms)Flushing d49177ea61caad30804146ea5c80f2ab/c: appending metadata at 1732634977506 (+14 ms)Flushing d49177ea61caad30804146ea5c80f2ab/c: closing flushed file at 1732634977506Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fa3145b: reopening flushed file at 1732634977516 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44f985d: reopening flushed file at 1732634977522 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4264ddf6: reopening flushed file at 1732634977526 (+4 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d49177ea61caad30804146ea5c80f2ab in 89ms, sequenceid=66, compaction requested=false; wal=null at 1732634977532 (+6 ms)Cleaning up temporary data from old regions at 1732634977534 (+2 ms)Region opened successfully at 1732634977539 (+5 ms) 2024-11-26T15:29:37,549 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d49177ea61caad30804146ea5c80f2ab, disabling compactions & flushes 2024-11-26T15:29:37,549 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:37,549 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:37,549 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. after waiting 0 ms 2024-11-26T15:29:37,549 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:37,550 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732634973025.d49177ea61caad30804146ea5c80f2ab. 2024-11-26T15:29:37,550 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d49177ea61caad30804146ea5c80f2ab: Waiting for close lock at 1732634977549Disabling compacts and flushes for region at 1732634977549Disabling writes for close at 1732634977549Writing region close event to WAL at 1732634977550 (+1 ms)Closed at 1732634977550 2024-11-26T15:29:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741939_1119 (size=93) 2024-11-26T15:29:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741939_1119 (size=93) 2024-11-26T15:29:37,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741939_1119 (size=93) 2024-11-26T15:29:37,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T15:29:37,555 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732634977401) 2024-11-26T15:29:37,569 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443 (was 439) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1311943922_22 at /127.0.0.1:39752 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:37937 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1311943922_22 at /127.0.0.1:56624 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:37937 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1311943922_22 at /127.0.0.1:54470 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=161 (was 175), ProcessCount=11 (was 11), AvailableMemoryMB=9313 (was 9330) 2024-11-26T15:29:37,569 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-26T15:29:37,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T15:29:37,569 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:29:37,569 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:29:37,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:37,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:37,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T15:29:37,570 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T15:29:37,570 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1107730837, stopped=false 2024-11-26T15:29:37,571 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6cc2bc8bf733,44737,1732634937484 2024-11-26T15:29:37,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T15:29:37,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T15:29:37,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T15:29:37,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:37,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:37,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:37,678 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T15:29:37,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:29:37,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:29:37,678 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T15:29:37,679 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:29:37,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:37,679 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6cc2bc8bf733,41249,1732634938354' ***** 2024-11-26T15:29:37,679 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T15:29:37,679 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T15:29:37,679 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6cc2bc8bf733,42689,1732634938394' ***** 2024-11-26T15:29:37,679 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T15:29:37,679 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T15:29:37,679 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T15:29:37,679 INFO [RS:2;6cc2bc8bf733:42689 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T15:29:37,679 INFO [RS:2;6cc2bc8bf733:42689 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(959): stopping server 6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T15:29:37,680 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;6cc2bc8bf733:42689. 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T15:29:37,680 DEBUG [RS:2;6cc2bc8bf733:42689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(3091): Received CLOSE for 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:37,680 DEBUG [RS:2;6cc2bc8bf733:42689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(959): stopping server 6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6cc2bc8bf733:41249. 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T15:29:37,680 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1122cfb9a9b693d960440938ac541ca4, disabling compactions & flushes 2024-11-26T15:29:37,680 DEBUG [RS:1;6cc2bc8bf733:41249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T15:29:37,680 DEBUG [RS:1;6cc2bc8bf733:41249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:37,680 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:37,680 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:37,680 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T15:29:37,680 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. after waiting 0 ms 2024-11-26T15:29:37,680 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1325): Online Regions={1122cfb9a9b693d960440938ac541ca4=testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4.} 2024-11-26T15:29:37,680 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:37,680 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T15:29:37,680 DEBUG [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1351): Waiting on 1122cfb9a9b693d960440938ac541ca4 2024-11-26T15:29:37,680 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T15:29:37,680 DEBUG [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T15:29:37,680 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T15:29:37,681 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T15:29:37,681 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T15:29:37,681 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T15:29:37,681 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T15:29:37,681 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-26T15:29:37,685 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1122cfb9a9b693d960440938ac541ca4/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-26T15:29:37,686 INFO [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:37,686 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1122cfb9a9b693d960440938ac541ca4: Waiting for close lock at 1732634977680Running coprocessor pre-close hooks at 1732634977680Disabling compacts and flushes for region at 1732634977680Disabling writes for close at 1732634977680Writing region close event to WAL at 1732634977682 (+2 ms)Running coprocessor post-close hooks at 1732634977685 (+3 ms)Closed at 1732634977685 2024-11-26T15:29:37,686 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/info/3e7371d529fc47d5b63b7e214082e7d5 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4./info:regioninfo/1732634956441/Put/seqid=0 2024-11-26T15:29:37,686 DEBUG [RS_CLOSE_REGION-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732634952370.1122cfb9a9b693d960440938ac541ca4. 2024-11-26T15:29:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741943_1123 (size=6778) 2024-11-26T15:29:37,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741943_1123 (size=6778) 2024-11-26T15:29:37,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741943_1123 (size=6778) 2024-11-26T15:29:37,696 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/info/3e7371d529fc47d5b63b7e214082e7d5 2024-11-26T15:29:37,702 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/.tmp/info/3e7371d529fc47d5b63b7e214082e7d5 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/info/3e7371d529fc47d5b63b7e214082e7d5 2024-11-26T15:29:37,707 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/info/3e7371d529fc47d5b63b7e214082e7d5, entries=8, sequenceid=23, filesize=6.6 K 2024-11-26T15:29:37,708 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=23, compaction requested=false 2024-11-26T15:29:37,712 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-26T15:29:37,712 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T15:29:37,712 INFO [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T15:29:37,712 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732634977680Running coprocessor pre-close hooks at 1732634977680Disabling compacts and flushes for region at 1732634977680Disabling writes for close at 1732634977681 (+1 ms)Obtaining lock to block concurrent updates at 1732634977681Preparing flush snapshotting stores in 1588230740 at 1732634977681Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732634977681Flushing stores of hbase:meta,,1.1588230740 at 1732634977682 (+1 ms)Flushing 1588230740/info: creating writer at 1732634977682Flushing 1588230740/info: appending metadata at 1732634977685 (+3 ms)Flushing 1588230740/info: closing flushed file at 1732634977685Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15e9ebd4: reopening flushed file at 1732634977701 (+16 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=23, compaction requested=false at 1732634977708 (+7 ms)Writing region close event to WAL at 1732634977709 (+1 ms)Running coprocessor post-close hooks at 1732634977712 (+3 ms)Closed at 1732634977712 2024-11-26T15:29:37,712 DEBUG [RS_CLOSE_META-regionserver/6cc2bc8bf733:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T15:29:37,738 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T15:29:37,738 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T15:29:37,767 INFO [regionserver/6cc2bc8bf733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:29:37,768 INFO [regionserver/6cc2bc8bf733:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:29:37,793 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T15:29:37,793 INFO [regionserver/6cc2bc8bf733:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T15:29:37,816 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-26T15:29:37,816 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:37,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-26T15:29:37,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-26T15:29:37,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T15:29:37,818 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T15:29:37,818 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T15:29:37,881 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(976): stopping server 6cc2bc8bf733,41249,1732634938354; all regions closed. 2024-11-26T15:29:37,881 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(976): stopping server 6cc2bc8bf733,42689,1732634938394; all regions closed. 2024-11-26T15:29:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741893_1071 (size=2422) 2024-11-26T15:29:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741834_1010 (size=3433) 2024-11-26T15:29:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741893_1071 (size=2422) 2024-11-26T15:29:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741834_1010 (size=3433) 2024-11-26T15:29:37,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741893_1071 (size=2422) 2024-11-26T15:29:37,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741834_1010 (size=3433) 2024-11-26T15:29:37,887 DEBUG [RS:1;6cc2bc8bf733:41249 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs 2024-11-26T15:29:37,887 INFO [RS:1;6cc2bc8bf733:41249 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6cc2bc8bf733%2C41249%2C1732634938354:(num 1732634939885) 2024-11-26T15:29:37,887 DEBUG [RS:1;6cc2bc8bf733:41249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:37,887 DEBUG [RS:2;6cc2bc8bf733:42689 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs 2024-11-26T15:29:37,888 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:29:37,888 INFO [RS:2;6cc2bc8bf733:42689 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6cc2bc8bf733%2C42689%2C1732634938394.meta:.meta(num 1732634955574) 2024-11-26T15:29:37,888 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T15:29:37,888 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.ChoreService(370): Chore service for: regionserver/6cc2bc8bf733:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T15:29:37,888 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T15:29:37,888 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T15:29:37,888 INFO [regionserver/6cc2bc8bf733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T15:29:37,888 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T15:29:37,888 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T15:29:37,889 INFO [RS:1;6cc2bc8bf733:41249 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41249 2024-11-26T15:29:37,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741835_1011 (size=93) 2024-11-26T15:29:37,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741835_1011 (size=93) 2024-11-26T15:29:37,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741835_1011 (size=93) 2024-11-26T15:29:37,893 DEBUG [RS:2;6cc2bc8bf733:42689 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/oldWALs 2024-11-26T15:29:37,893 INFO [RS:2;6cc2bc8bf733:42689 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6cc2bc8bf733%2C42689%2C1732634938394:(num 1732634939885) 2024-11-26T15:29:37,893 DEBUG [RS:2;6cc2bc8bf733:42689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T15:29:37,893 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T15:29:37,893 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T15:29:37,893 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.ChoreService(370): Chore service for: regionserver/6cc2bc8bf733:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T15:29:37,893 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T15:29:37,894 INFO [regionserver/6cc2bc8bf733:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T15:29:37,894 INFO [RS:2;6cc2bc8bf733:42689 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42689 2024-11-26T15:29:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6cc2bc8bf733,41249,1732634938354 2024-11-26T15:29:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T15:29:37,903 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T15:29:37,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6cc2bc8bf733,42689,1732634938394 2024-11-26T15:29:37,914 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T15:29:37,924 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6cc2bc8bf733,41249,1732634938354] 2024-11-26T15:29:37,945 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6cc2bc8bf733,41249,1732634938354 already deleted, retry=false 2024-11-26T15:29:37,946 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6cc2bc8bf733,41249,1732634938354 expired; onlineServers=1 2024-11-26T15:29:37,946 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6cc2bc8bf733,42689,1732634938394] 2024-11-26T15:29:37,956 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6cc2bc8bf733,42689,1732634938394 already deleted, retry=false 2024-11-26T15:29:37,956 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6cc2bc8bf733,42689,1732634938394 expired; onlineServers=0 2024-11-26T15:29:37,956 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6cc2bc8bf733,44737,1732634937484' ***** 2024-11-26T15:29:37,956 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T15:29:37,957 INFO [M:0;6cc2bc8bf733:44737 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T15:29:37,957 INFO [M:0;6cc2bc8bf733:44737 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T15:29:37,957 DEBUG [M:0;6cc2bc8bf733:44737 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T15:29:37,957 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T15:29:37,957 DEBUG [M:0;6cc2bc8bf733:44737 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T15:29:37,957 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.large.0-1732634939513 {}] cleaner.HFileCleaner(306): Exit Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.large.0-1732634939513,5,FailOnTimeoutGroup] 2024-11-26T15:29:37,957 DEBUG [master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.small.0-1732634939521 {}] cleaner.HFileCleaner(306): Exit Thread[master/6cc2bc8bf733:0:becomeActiveMaster-HFileCleaner.small.0-1732634939521,5,FailOnTimeoutGroup] 2024-11-26T15:29:37,958 INFO [M:0;6cc2bc8bf733:44737 {}] hbase.ChoreService(370): Chore service for: master/6cc2bc8bf733:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T15:29:37,958 INFO [M:0;6cc2bc8bf733:44737 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T15:29:37,958 DEBUG [M:0;6cc2bc8bf733:44737 {}] master.HMaster(1795): Stopping service threads 2024-11-26T15:29:37,959 INFO [M:0;6cc2bc8bf733:44737 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T15:29:37,959 INFO [M:0;6cc2bc8bf733:44737 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T15:29:37,960 INFO [M:0;6cc2bc8bf733:44737 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T15:29:37,961 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T15:29:37,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T15:29:37,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T15:29:37,967 DEBUG [M:0;6cc2bc8bf733:44737 {}] zookeeper.ZKUtil(347): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T15:29:37,967 WARN [M:0;6cc2bc8bf733:44737 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T15:29:37,967 INFO [M:0;6cc2bc8bf733:44737 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/.lastflushedseqids 2024-11-26T15:29:37,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741944_1124 (size=119) 2024-11-26T15:29:37,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741944_1124 (size=119) 2024-11-26T15:29:37,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741944_1124 (size=119) 2024-11-26T15:29:37,980 INFO [M:0;6cc2bc8bf733:44737 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T15:29:37,981 INFO [M:0;6cc2bc8bf733:44737 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T15:29:37,981 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T15:29:37,981 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:29:37,981 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:29:37,981 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T15:29:37,981 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:29:37,981 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-26T15:29:38,018 DEBUG [M:0;6cc2bc8bf733:44737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/32dbf9b7ffe14a96a1629dbb34cd4ae9 is 82, key is hbase:meta,,1/info:regioninfo/1732634955729/Put/seqid=0 2024-11-26T15:29:38,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741945_1125 (size=6063) 2024-11-26T15:29:38,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741945_1125 (size=6063) 2024-11-26T15:29:38,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741945_1125 (size=6063) 2024-11-26T15:29:38,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:38,024 INFO [RS:1;6cc2bc8bf733:41249 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T15:29:38,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41249-0x10177c301390002, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:38,024 INFO [RS:1;6cc2bc8bf733:41249 {}] regionserver.HRegionServer(1031): Exiting; stopping=6cc2bc8bf733,41249,1732634938354; zookeeper connection closed. 2024-11-26T15:29:38,025 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5065e163 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5065e163 2024-11-26T15:29:38,026 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/32dbf9b7ffe14a96a1629dbb34cd4ae9 2024-11-26T15:29:38,035 INFO [RS:2;6cc2bc8bf733:42689 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T15:29:38,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:38,035 INFO [RS:2;6cc2bc8bf733:42689 {}] regionserver.HRegionServer(1031): Exiting; stopping=6cc2bc8bf733,42689,1732634938394; zookeeper connection closed. 2024-11-26T15:29:38,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42689-0x10177c301390003, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:38,035 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3286483b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3286483b 2024-11-26T15:29:38,036 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T15:29:38,044 DEBUG [M:0;6cc2bc8bf733:44737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b399276b254b41d1a5712f6d9103993c is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732634952802/Put/seqid=0 2024-11-26T15:29:38,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741946_1126 (size=7907) 2024-11-26T15:29:38,052 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b399276b254b41d1a5712f6d9103993c 2024-11-26T15:29:38,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741946_1126 (size=7907) 2024-11-26T15:29:38,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741946_1126 (size=7907) 2024-11-26T15:29:38,056 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b399276b254b41d1a5712f6d9103993c 2024-11-26T15:29:38,071 DEBUG [M:0;6cc2bc8bf733:44737 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/388df3aee216402fb3d43eb9d996fb6a is 69, key is 6cc2bc8bf733,41249,1732634938354/rs:state/1732634939608/Put/seqid=0 2024-11-26T15:29:38,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741947_1127 (size=5440) 2024-11-26T15:29:38,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741947_1127 (size=5440) 2024-11-26T15:29:38,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741947_1127 (size=5440) 2024-11-26T15:29:38,077 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/388df3aee216402fb3d43eb9d996fb6a 2024-11-26T15:29:38,082 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 388df3aee216402fb3d43eb9d996fb6a 2024-11-26T15:29:38,083 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/32dbf9b7ffe14a96a1629dbb34cd4ae9 as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/32dbf9b7ffe14a96a1629dbb34cd4ae9 2024-11-26T15:29:38,088 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/32dbf9b7ffe14a96a1629dbb34cd4ae9, entries=14, sequenceid=207, filesize=5.9 K 2024-11-26T15:29:38,090 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b399276b254b41d1a5712f6d9103993c as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b399276b254b41d1a5712f6d9103993c 2024-11-26T15:29:38,094 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b399276b254b41d1a5712f6d9103993c 2024-11-26T15:29:38,095 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b399276b254b41d1a5712f6d9103993c, entries=21, sequenceid=207, filesize=7.7 K 2024-11-26T15:29:38,096 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/388df3aee216402fb3d43eb9d996fb6a as hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/388df3aee216402fb3d43eb9d996fb6a 2024-11-26T15:29:38,100 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 388df3aee216402fb3d43eb9d996fb6a 2024-11-26T15:29:38,100 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37937/user/jenkins/test-data/220105e3-c820-1804-7c81-9ff1cfd5bef7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/388df3aee216402fb3d43eb9d996fb6a, entries=3, sequenceid=207, filesize=5.3 K 2024-11-26T15:29:38,102 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=207, compaction requested=false 2024-11-26T15:29:38,103 INFO [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T15:29:38,103 DEBUG [M:0;6cc2bc8bf733:44737 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732634977981Disabling compacts and flushes for region at 1732634977981Disabling writes for close at 1732634977981Obtaining lock to block concurrent updates at 1732634977981Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732634977981Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1732634977981Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732634977982 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732634977982Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732634978017 (+35 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732634978017Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732634978030 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732634978044 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732634978044Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732634978057 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732634978070 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732634978070Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b160ec4: reopening flushed file at 1732634978082 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51f2cda4: reopening flushed file at 1732634978089 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@121e53e5: reopening flushed file at 1732634978095 (+6 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=207, compaction requested=false at 1732634978102 (+7 ms)Writing region close event to WAL at 1732634978103 (+1 ms)Closed at 1732634978103 2024-11-26T15:29:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39747 is added to blk_1073741830_1006 (size=99436) 2024-11-26T15:29:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35535 is added to blk_1073741830_1006 (size=99436) 2024-11-26T15:29:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741830_1006 (size=99436) 2024-11-26T15:29:38,108 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T15:29:38,108 INFO [M:0;6cc2bc8bf733:44737 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T15:29:38,108 INFO [M:0;6cc2bc8bf733:44737 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44737 2024-11-26T15:29:38,108 INFO [M:0;6cc2bc8bf733:44737 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T15:29:38,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:38,214 INFO [M:0;6cc2bc8bf733:44737 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T15:29:38,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44737-0x10177c301390000, quorum=127.0.0.1:60921, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T15:29:38,222 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634967244 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634967244 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732634957912/wal.1732634967244 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T15:29:38,226 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957767 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957767 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732634956986/wal.1732634957767 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T15:29:38,227 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732634947930/wal.1732634948034 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:38,229 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634968090 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634968090 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732634967483/wal.1732634968090 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T15:29:38,230 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732634968233/wal.1732634968287 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:38,231 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732634973024/wal.1732634973208 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:38,231 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732634941549/wal.1732634941619 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T15:29:38,233 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal.1732634941422 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal.1732634941422 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732634941240/wal.1732634941422 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T15:29:38,236 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal.1732634940987 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal.1732634940987 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732634940738/wal.1732634940987 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1674615901_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T15:29:38,276 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:29:38,281 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T15:29:38,281 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T15:29:38,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T15:29:38,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,STOPPED} 2024-11-26T15:29:38,285 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T15:29:38,285 WARN [BP-814321108-172.17.0.3-1732634932847 heartbeating to localhost/127.0.0.1:37937 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T15:29:38,285 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T15:29:38,285 WARN [BP-814321108-172.17.0.3-1732634932847 heartbeating to localhost/127.0.0.1:37937 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-814321108-172.17.0.3-1732634932847 (Datanode Uuid e94b488b-994a-402a-8fa1-f753723bc7c1) service to localhost/127.0.0.1:37937 2024-11-26T15:29:38,287 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data5/current/BP-814321108-172.17.0.3-1732634932847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:29:38,287 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data6/current/BP-814321108-172.17.0.3-1732634932847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:29:38,288 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T15:29:38,290 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:29:38,290 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T15:29:38,290 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T15:29:38,290 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T15:29:38,290 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,STOPPED} 2024-11-26T15:29:38,291 WARN [BP-814321108-172.17.0.3-1732634932847 heartbeating to localhost/127.0.0.1:37937 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T15:29:38,291 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T15:29:38,291 WARN [BP-814321108-172.17.0.3-1732634932847 heartbeating to localhost/127.0.0.1:37937 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-814321108-172.17.0.3-1732634932847 (Datanode Uuid 2110294a-95fc-49c4-bdc4-7f402fcbd37b) service to localhost/127.0.0.1:37937 2024-11-26T15:29:38,291 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T15:29:38,292 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data3/current/BP-814321108-172.17.0.3-1732634932847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:29:38,292 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data4/current/BP-814321108-172.17.0.3-1732634932847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:29:38,292 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T15:29:38,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T15:29:38,297 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T15:29:38,297 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T15:29:38,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T15:29:38,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,STOPPED} 2024-11-26T15:29:38,298 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T15:29:38,298 WARN [BP-814321108-172.17.0.3-1732634932847 heartbeating to localhost/127.0.0.1:37937 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T15:29:38,299 WARN [BP-814321108-172.17.0.3-1732634932847 heartbeating to localhost/127.0.0.1:37937 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-814321108-172.17.0.3-1732634932847 (Datanode Uuid 405b8008-fac7-42f3-91b5-9ae72de4f169) service to localhost/127.0.0.1:37937 2024-11-26T15:29:38,299 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T15:29:38,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data1/current/BP-814321108-172.17.0.3-1732634932847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:29:38,299 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/cluster_9f310ea5-59ca-2823-852d-d13ce1a4fbba/data/data2/current/BP-814321108-172.17.0.3-1732634932847 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T15:29:38,299 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T15:29:38,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T15:29:38,305 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T15:29:38,305 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T15:29:38,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T15:29:38,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d743b0b0-aaea-ae2e-4bff-325ea37145f0/hadoop.log.dir/,STOPPED} 2024-11-26T15:29:38,314 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T15:29:38,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down