2024-12-11 22:58:25,181 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-11 22:58:25,196 main DEBUG Took 0.012642 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 22:58:25,196 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 22:58:25,196 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 22:58:25,198 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 22:58:25,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,206 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 22:58:25,217 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,219 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,220 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,220 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,221 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,222 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,222 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,222 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,223 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,223 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,224 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,224 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,225 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,226 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,226 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,226 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,227 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:58:25,227 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,228 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 22:58:25,229 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:58:25,230 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 22:58:25,231 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 22:58:25,232 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 22:58:25,233 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 22:58:25,233 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 22:58:25,241 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 22:58:25,244 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 22:58:25,245 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 22:58:25,245 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 22:58:25,246 main DEBUG createAppenders(={Console}) 2024-12-11 22:58:25,246 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-12-11 22:58:25,247 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-11 22:58:25,247 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-12-11 22:58:25,248 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 22:58:25,248 main DEBUG OutputStream closed 2024-12-11 22:58:25,248 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 22:58:25,248 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 22:58:25,249 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-12-11 22:58:25,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 22:58:25,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 22:58:25,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 22:58:25,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 22:58:25,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 22:58:25,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 22:58:25,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 22:58:25,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 22:58:25,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 22:58:25,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 22:58:25,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 22:58:25,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 22:58:25,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 22:58:25,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 22:58:25,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 22:58:25,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 22:58:25,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 22:58:25,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 22:58:25,324 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 22:58:25,325 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-12-11 22:58:25,325 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 22:58:25,325 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-12-11T22:58:25,571 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53 2024-12-11 22:58:25,574 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 22:58:25,574 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T22:58:25,583 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-11T22:58:25,609 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T22:58:25,653 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T22:58:25,654 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T22:58:25,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-11T22:58:25,682 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028, deleteOnExit=true 2024-12-11T22:58:25,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-11T22:58:25,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/test.cache.data in system properties and HBase conf 2024-12-11T22:58:25,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.tmp.dir in system properties and HBase conf 2024-12-11T22:58:25,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir in system properties and HBase conf 2024-12-11T22:58:25,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-11T22:58:25,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-11T22:58:25,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-11T22:58:25,772 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T22:58:25,869 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:58:25,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-11T22:58:25,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-11T22:58:25,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-11T22:58:25,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T22:58:25,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-11T22:58:25,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-11T22:58:25,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T22:58:25,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T22:58:25,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-11T22:58:25,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/nfs.dump.dir in system properties and HBase conf 2024-12-11T22:58:25,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/java.io.tmpdir in system properties and HBase conf 2024-12-11T22:58:25,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T22:58:25,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-11T22:58:25,881 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-11T22:58:26,703 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-11T22:58:26,778 INFO [Time-limited test {}] log.Log(170): Logging initialized @2328ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-11T22:58:26,864 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:58:26,922 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:58:26,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:58:26,947 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:58:26,948 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T22:58:26,970 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:58:26,973 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39b35319{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:58:26,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@377439aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:58:27,163 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25dc94e0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/java.io.tmpdir/jetty-localhost-46729-hadoop-hdfs-3_4_1-tests_jar-_-any-86428047550194199/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T22:58:27,170 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7cc108fc{HTTP/1.1, (http/1.1)}{localhost:46729} 2024-12-11T22:58:27,170 INFO [Time-limited test {}] server.Server(415): Started @2720ms 2024-12-11T22:58:27,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:58:27,570 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:58:27,573 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:58:27,573 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:58:27,573 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T22:58:27,574 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@734853a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:58:27,575 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74576bd2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:58:27,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c65e35b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/java.io.tmpdir/jetty-localhost-36801-hadoop-hdfs-3_4_1-tests_jar-_-any-5141738315083343574/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:58:27,697 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@67cd2c22{HTTP/1.1, (http/1.1)}{localhost:36801} 2024-12-11T22:58:27,698 INFO [Time-limited test {}] server.Server(415): Started @3247ms 2024-12-11T22:58:27,751 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T22:58:27,875 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:58:27,882 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:58:27,884 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:58:27,884 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:58:27,884 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-11T22:58:27,887 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1919f676{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:58:27,887 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@77f84511{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:58:28,025 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@36c913bb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/java.io.tmpdir/jetty-localhost-41895-hadoop-hdfs-3_4_1-tests_jar-_-any-18023226892967782618/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:58:28,026 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a0e4ecc{HTTP/1.1, (http/1.1)}{localhost:41895} 2024-12-11T22:58:28,026 INFO [Time-limited test {}] server.Server(415): Started @3576ms 2024-12-11T22:58:28,029 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T22:58:28,071 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T22:58:28,077 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T22:58:28,081 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T22:58:28,081 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T22:58:28,081 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-11T22:58:28,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4752d0b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,AVAILABLE} 2024-12-11T22:58:28,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e5ef071{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T22:58:28,212 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3e2333e3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/java.io.tmpdir/jetty-localhost-45413-hadoop-hdfs-3_4_1-tests_jar-_-any-9594432560212089449/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:58:28,213 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data2/current/BP-1204600519-172.17.0.3-1733957906467/current, will proceed with Du for space computation calculation, 2024-12-11T22:58:28,213 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data3/current/BP-1204600519-172.17.0.3-1733957906467/current, will proceed with Du for space computation calculation, 2024-12-11T22:58:28,213 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data1/current/BP-1204600519-172.17.0.3-1733957906467/current, will proceed with Du for space computation calculation, 2024-12-11T22:58:28,213 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data4/current/BP-1204600519-172.17.0.3-1733957906467/current, will proceed with Du for space computation calculation, 2024-12-11T22:58:28,213 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b860b3d{HTTP/1.1, (http/1.1)}{localhost:45413} 2024-12-11T22:58:28,214 INFO [Time-limited test {}] server.Server(415): Started @3763ms 2024-12-11T22:58:28,217 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T22:58:28,274 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T22:58:28,275 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T22:58:28,343 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data5/current/BP-1204600519-172.17.0.3-1733957906467/current, will proceed with Du for space computation calculation, 2024-12-11T22:58:28,343 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data6/current/BP-1204600519-172.17.0.3-1733957906467/current, will proceed with Du for space computation calculation, 2024-12-11T22:58:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x614c40e5ffd842b2 with lease ID 0x38928a7ddde4c6eb: Processing first storage report for DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4 from datanode DatanodeRegistration(127.0.0.1:39291, datanodeUuid=0d606b3a-f8db-4a97-a721-5950cf4fad3a, infoPort=35233, infoSecurePort=0, ipcPort=43201, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467) 2024-12-11T22:58:28,364 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x614c40e5ffd842b2 with lease ID 0x38928a7ddde4c6eb: from storage DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4 node DatanodeRegistration(127.0.0.1:39291, datanodeUuid=0d606b3a-f8db-4a97-a721-5950cf4fad3a, infoPort=35233, infoSecurePort=0, ipcPort=43201, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-11T22:58:28,365 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x614c40e5ffd842b2 with lease ID 0x38928a7ddde4c6eb: Processing first storage report for DS-79e4911b-8593-4af0-a91e-d4a02eefa1f0 from datanode DatanodeRegistration(127.0.0.1:39291, datanodeUuid=0d606b3a-f8db-4a97-a721-5950cf4fad3a, infoPort=35233, infoSecurePort=0, ipcPort=43201, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467) 2024-12-11T22:58:28,365 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x614c40e5ffd842b2 with lease ID 0x38928a7ddde4c6eb: from storage DS-79e4911b-8593-4af0-a91e-d4a02eefa1f0 node DatanodeRegistration(127.0.0.1:39291, datanodeUuid=0d606b3a-f8db-4a97-a721-5950cf4fad3a, infoPort=35233, infoSecurePort=0, ipcPort=43201, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T22:58:28,366 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb75031651443c85a with lease ID 0x38928a7ddde4c6ec: Processing first storage report for DS-6c1894a9-6b42-4116-a0c8-df45eb92e533 from datanode DatanodeRegistration(127.0.0.1:37177, datanodeUuid=71b54c86-2458-4d06-9d91-d41840b55c89, infoPort=43745, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467) 2024-12-11T22:58:28,367 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb75031651443c85a with lease ID 0x38928a7ddde4c6ec: from storage DS-6c1894a9-6b42-4116-a0c8-df45eb92e533 node DatanodeRegistration(127.0.0.1:37177, datanodeUuid=71b54c86-2458-4d06-9d91-d41840b55c89, infoPort=43745, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T22:58:28,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb75031651443c85a with lease ID 0x38928a7ddde4c6ec: Processing first storage report for DS-4d80c07e-1c9a-4f78-975f-330753b243cb from datanode DatanodeRegistration(127.0.0.1:37177, datanodeUuid=71b54c86-2458-4d06-9d91-d41840b55c89, infoPort=43745, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467) 2024-12-11T22:58:28,367 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb75031651443c85a with lease ID 0x38928a7ddde4c6ec: from storage DS-4d80c07e-1c9a-4f78-975f-330753b243cb node DatanodeRegistration(127.0.0.1:37177, datanodeUuid=71b54c86-2458-4d06-9d91-d41840b55c89, infoPort=43745, infoSecurePort=0, ipcPort=42893, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T22:58:28,383 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T22:58:28,389 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6fd6ec49befe66b8 with lease ID 0x38928a7ddde4c6ed: Processing first storage report for DS-af6d2772-c851-4c16-96c2-835a220a12da from datanode DatanodeRegistration(127.0.0.1:43249, datanodeUuid=422e996f-6fdd-435a-bd71-360e657922dd, infoPort=40895, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467) 2024-12-11T22:58:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6fd6ec49befe66b8 with lease ID 0x38928a7ddde4c6ed: from storage DS-af6d2772-c851-4c16-96c2-835a220a12da node DatanodeRegistration(127.0.0.1:43249, datanodeUuid=422e996f-6fdd-435a-bd71-360e657922dd, infoPort=40895, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-11T22:58:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6fd6ec49befe66b8 with lease ID 0x38928a7ddde4c6ed: Processing first storage report for DS-0eee9d76-88df-41bc-896a-1b64b9ffce01 from datanode DatanodeRegistration(127.0.0.1:43249, datanodeUuid=422e996f-6fdd-435a-bd71-360e657922dd, infoPort=40895, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467) 2024-12-11T22:58:28,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6fd6ec49befe66b8 with lease ID 0x38928a7ddde4c6ed: from storage DS-0eee9d76-88df-41bc-896a-1b64b9ffce01 node DatanodeRegistration(127.0.0.1:43249, datanodeUuid=422e996f-6fdd-435a-bd71-360e657922dd, infoPort=40895, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=2039544363;c=1733957906467), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T22:58:28,601 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53 2024-12-11T22:58:28,675 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/zookeeper_0, clientPort=52314, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T22:58:28,684 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52314 2024-12-11T22:58:28,698 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:28,713 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:28,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741825_1001 (size=7) 2024-12-11T22:58:28,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741825_1001 (size=7) 2024-12-11T22:58:28,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741825_1001 (size=7) 2024-12-11T22:58:29,333 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c with version=8 2024-12-11T22:58:29,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/hbase-staging 2024-12-11T22:58:29,682 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b48762765ec4:0 server-side Connection retries=45 2024-12-11T22:58:29,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:29,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:29,698 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:58:29,698 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:29,698 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:58:29,833 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T22:58:29,893 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T22:58:29,902 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T22:58:29,906 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:58:29,935 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2783 (auto-detected) 2024-12-11T22:58:29,936 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-11T22:58:29,956 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38869 2024-12-11T22:58:29,977 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38869 connecting to ZooKeeper ensemble=127.0.0.1:52314 2024-12-11T22:58:30,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388690x0, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:58:30,011 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38869-0x100cb900b300000 connected 2024-12-11T22:58:30,038 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,042 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,052 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:58:30,056 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c, hbase.cluster.distributed=false 2024-12-11T22:58:30,078 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:58:30,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38869 2024-12-11T22:58:30,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38869 2024-12-11T22:58:30,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38869 2024-12-11T22:58:30,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38869 2024-12-11T22:58:30,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38869 2024-12-11T22:58:30,196 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b48762765ec4:0 server-side Connection retries=45 2024-12-11T22:58:30,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,199 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:58:30,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:58:30,202 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T22:58:30,204 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:58:30,205 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37687 2024-12-11T22:58:30,206 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37687 connecting to ZooKeeper ensemble=127.0.0.1:52314 2024-12-11T22:58:30,208 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,210 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376870x0, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:58:30,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:376870x0, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:58:30,221 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37687-0x100cb900b300001 connected 2024-12-11T22:58:30,224 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T22:58:30,231 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T22:58:30,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T22:58:30,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:58:30,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37687 2024-12-11T22:58:30,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37687 2024-12-11T22:58:30,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37687 2024-12-11T22:58:30,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37687 2024-12-11T22:58:30,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37687 2024-12-11T22:58:30,259 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b48762765ec4:0 server-side Connection retries=45 2024-12-11T22:58:30,259 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,259 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,260 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:58:30,260 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,260 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:58:30,260 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T22:58:30,260 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:58:30,261 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34571 2024-12-11T22:58:30,263 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34571 connecting to ZooKeeper ensemble=127.0.0.1:52314 2024-12-11T22:58:30,264 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,266 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:345710x0, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:58:30,272 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:345710x0, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:58:30,272 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34571-0x100cb900b300002 connected 2024-12-11T22:58:30,273 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T22:58:30,275 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T22:58:30,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T22:58:30,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:58:30,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34571 2024-12-11T22:58:30,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34571 2024-12-11T22:58:30,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34571 2024-12-11T22:58:30,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34571 2024-12-11T22:58:30,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34571 2024-12-11T22:58:30,297 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b48762765ec4:0 server-side Connection retries=45 2024-12-11T22:58:30,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,298 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T22:58:30,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T22:58:30,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T22:58:30,298 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T22:58:30,299 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T22:58:30,299 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33507 2024-12-11T22:58:30,301 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33507 connecting to ZooKeeper ensemble=127.0.0.1:52314 2024-12-11T22:58:30,302 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,305 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335070x0, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T22:58:30,311 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:335070x0, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:58:30,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33507-0x100cb900b300003 connected 2024-12-11T22:58:30,312 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T22:58:30,316 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T22:58:30,317 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T22:58:30,319 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T22:58:30,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33507 2024-12-11T22:58:30,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33507 2024-12-11T22:58:30,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33507 2024-12-11T22:58:30,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33507 2024-12-11T22:58:30,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33507 2024-12-11T22:58:30,338 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b48762765ec4:38869 2024-12-11T22:58:30,340 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b48762765ec4,38869,1733957909474 2024-12-11T22:58:30,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,351 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b48762765ec4,38869,1733957909474 2024-12-11T22:58:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T22:58:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T22:58:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T22:58:30,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,375 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T22:58:30,376 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b48762765ec4,38869,1733957909474 from backup master directory 2024-12-11T22:58:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b48762765ec4,38869,1733957909474 2024-12-11T22:58:30,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T22:58:30,381 WARN [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:58:30,381 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b48762765ec4,38869,1733957909474 2024-12-11T22:58:30,383 INFO [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T22:58:30,384 INFO [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T22:58:30,452 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/hbase.id] with ID: 55e41239-6683-4c79-8b12-6cb06edb1ab3 2024-12-11T22:58:30,452 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/.tmp/hbase.id 2024-12-11T22:58:30,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741826_1002 (size=42) 2024-12-11T22:58:30,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741826_1002 (size=42) 2024-12-11T22:58:30,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741826_1002 (size=42) 2024-12-11T22:58:30,471 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/.tmp/hbase.id]:[hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/hbase.id] 2024-12-11T22:58:30,520 INFO [master/b48762765ec4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T22:58:30,525 INFO [master/b48762765ec4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T22:58:30,545 INFO [master/b48762765ec4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-11T22:58:30,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:30,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741827_1003 (size=196) 2024-12-11T22:58:30,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741827_1003 (size=196) 2024-12-11T22:58:30,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741827_1003 (size=196) 2024-12-11T22:58:30,589 INFO [master/b48762765ec4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T22:58:30,591 INFO [master/b48762765ec4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T22:58:30,598 INFO [master/b48762765ec4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:30,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741828_1004 (size=1189) 2024-12-11T22:58:30,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741828_1004 (size=1189) 2024-12-11T22:58:30,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741828_1004 (size=1189) 2024-12-11T22:58:30,655 INFO [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store 2024-12-11T22:58:30,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741829_1005 (size=34) 2024-12-11T22:58:30,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741829_1005 (size=34) 2024-12-11T22:58:30,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741829_1005 (size=34) 2024-12-11T22:58:30,687 INFO [master/b48762765ec4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T22:58:30,691 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:30,692 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T22:58:30,693 INFO [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:58:30,693 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:58:30,695 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T22:58:30,695 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:58:30,695 INFO [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:58:30,697 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733957910692Disabling compacts and flushes for region at 1733957910692Disabling writes for close at 1733957910695 (+3 ms)Writing region close event to WAL at 1733957910695Closed at 1733957910695 2024-12-11T22:58:30,699 WARN [master/b48762765ec4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/.initializing 2024-12-11T22:58:30,700 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/WALs/b48762765ec4,38869,1733957909474 2024-12-11T22:58:30,710 INFO [master/b48762765ec4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:30,727 INFO [master/b48762765ec4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b48762765ec4%2C38869%2C1733957909474, suffix=, logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/WALs/b48762765ec4,38869,1733957909474, archiveDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/oldWALs, maxLogs=10 2024-12-11T22:58:30,771 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/WALs/b48762765ec4,38869,1733957909474/b48762765ec4%2C38869%2C1733957909474.1733957910733, exclude list is [], retry=0 2024-12-11T22:58:30,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:30,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:30,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:30,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:30,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-11T22:58:30,845 INFO [master/b48762765ec4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/WALs/b48762765ec4,38869,1733957909474/b48762765ec4%2C38869%2C1733957909474.1733957910733 2024-12-11T22:58:30,847 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:30,847 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:30,848 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:30,851 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,853 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,898 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T22:58:30,929 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:30,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:30,933 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,937 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T22:58:30,937 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:30,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:30,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,942 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T22:58:30,942 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:30,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:30,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,946 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T22:58:30,947 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:30,948 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:30,948 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,952 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,953 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,958 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,959 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,962 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:58:30,966 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T22:58:30,971 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:30,972 INFO [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62643849, jitterRate=-0.06653390824794769}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:58:30,979 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733957910868Initializing all the Stores at 1733957910871 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957910871Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957910872 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957910872Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957910872Cleaning up temporary data from old regions at 1733957910959 (+87 ms)Region opened successfully at 1733957910979 (+20 ms) 2024-12-11T22:58:30,985 INFO [master/b48762765ec4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T22:58:31,019 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357bb142, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b48762765ec4/172.17.0.3:0 2024-12-11T22:58:31,052 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T22:58:31,063 INFO [master/b48762765ec4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T22:58:31,063 INFO [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T22:58:31,066 INFO [master/b48762765ec4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T22:58:31,068 INFO [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-11T22:58:31,073 INFO [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T22:58:31,073 INFO [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T22:58:31,098 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T22:58:31,106 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T22:58:31,108 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T22:58:31,111 INFO [master/b48762765ec4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T22:58:31,112 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T22:58:31,114 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T22:58:31,116 INFO [master/b48762765ec4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T22:58:31,119 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T22:58:31,120 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T22:58:31,122 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T22:58:31,123 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T22:58:31,140 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T22:58:31,141 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T22:58:31,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:58:31,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:58:31,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:58:31,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T22:58:31,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,148 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b48762765ec4,38869,1733957909474, sessionid=0x100cb900b300000, setting cluster-up flag (Was=false) 2024-12-11T22:58:31,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,170 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T22:58:31,171 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b48762765ec4,38869,1733957909474 2024-12-11T22:58:31,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,182 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T22:58:31,184 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b48762765ec4,38869,1733957909474 2024-12-11T22:58:31,190 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T22:58:31,226 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(746): ClusterId : 55e41239-6683-4c79-8b12-6cb06edb1ab3 2024-12-11T22:58:31,226 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(746): ClusterId : 55e41239-6683-4c79-8b12-6cb06edb1ab3 2024-12-11T22:58:31,226 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(746): ClusterId : 55e41239-6683-4c79-8b12-6cb06edb1ab3 2024-12-11T22:58:31,229 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T22:58:31,229 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T22:58:31,229 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T22:58:31,234 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T22:58:31,234 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T22:58:31,234 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T22:58:31,235 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T22:58:31,235 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T22:58:31,235 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T22:58:31,238 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T22:58:31,238 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T22:58:31,238 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T22:58:31,239 DEBUG [RS:0;b48762765ec4:37687 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b48762765ec4/172.17.0.3:0 2024-12-11T22:58:31,239 DEBUG [RS:1;b48762765ec4:34571 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c12ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b48762765ec4/172.17.0.3:0 2024-12-11T22:58:31,239 DEBUG [RS:2;b48762765ec4:33507 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d314cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b48762765ec4/172.17.0.3:0 2024-12-11T22:58:31,257 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b48762765ec4:37687 2024-12-11T22:58:31,257 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b48762765ec4:34571 2024-12-11T22:58:31,260 INFO [RS:0;b48762765ec4:37687 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T22:58:31,260 INFO [RS:1;b48762765ec4:34571 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T22:58:31,260 INFO [RS:0;b48762765ec4:37687 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T22:58:31,260 INFO [RS:1;b48762765ec4:34571 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T22:58:31,260 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T22:58:31,261 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T22:58:31,261 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b48762765ec4:33507 2024-12-11T22:58:31,261 INFO [RS:2;b48762765ec4:33507 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T22:58:31,262 INFO [RS:2;b48762765ec4:33507 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T22:58:31,262 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T22:58:31,264 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(2659): reportForDuty to master=b48762765ec4,38869,1733957909474 with port=37687, startcode=1733957910157 2024-12-11T22:58:31,264 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(2659): reportForDuty to master=b48762765ec4,38869,1733957909474 with port=33507, startcode=1733957910297 2024-12-11T22:58:31,264 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(2659): reportForDuty to master=b48762765ec4,38869,1733957909474 with port=34571, startcode=1733957910258 2024-12-11T22:58:31,272 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T22:58:31,280 DEBUG [RS:0;b48762765ec4:37687 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T22:58:31,280 DEBUG [RS:1;b48762765ec4:34571 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T22:58:31,280 DEBUG [RS:2;b48762765ec4:33507 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T22:58:31,283 INFO [master/b48762765ec4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T22:58:31,291 INFO [master/b48762765ec4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T22:58:31,297 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b48762765ec4,38869,1733957909474 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T22:58:31,308 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b48762765ec4:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:58:31,309 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b48762765ec4:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:58:31,311 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b48762765ec4:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:58:31,311 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b48762765ec4:0, corePoolSize=5, maxPoolSize=5 2024-12-11T22:58:31,311 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b48762765ec4:0, corePoolSize=10, maxPoolSize=10 2024-12-11T22:58:31,311 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,312 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b48762765ec4:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:58:31,312 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,319 INFO [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733957941319 2024-12-11T22:58:31,320 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T22:58:31,322 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T22:58:31,326 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T22:58:31,326 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52471, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T22:58:31,327 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T22:58:31,327 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58763, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T22:58:31,328 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T22:58:31,328 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T22:58:31,328 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T22:58:31,328 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T22:58:31,332 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T22:58:31,334 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:58:31,339 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:31,339 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T22:58:31,335 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,343 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:58:31,344 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-11T22:58:31,345 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T22:58:31,346 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T22:58:31,347 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T22:58:31,349 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T22:58:31,350 INFO [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T22:58:31,352 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b48762765ec4:0:becomeActiveMaster-HFileCleaner.large.0-1733957911351,5,FailOnTimeoutGroup] 2024-12-11T22:58:31,356 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b48762765ec4:0:becomeActiveMaster-HFileCleaner.small.0-1733957911353,5,FailOnTimeoutGroup] 2024-12-11T22:58:31,357 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,357 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T22:58:31,358 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,358 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,379 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-11T22:58:31,379 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-11T22:58:31,379 WARN [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-11T22:58:31,379 WARN [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-11T22:58:31,379 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-11T22:58:31,379 WARN [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-11T22:58:31,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741831_1007 (size=1321) 2024-12-11T22:58:31,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741831_1007 (size=1321) 2024-12-11T22:58:31,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741831_1007 (size=1321) 2024-12-11T22:58:31,382 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T22:58:31,383 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c 2024-12-11T22:58:31,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741832_1008 (size=32) 2024-12-11T22:58:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741832_1008 (size=32) 2024-12-11T22:58:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741832_1008 (size=32) 2024-12-11T22:58:31,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:31,400 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T22:58:31,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T22:58:31,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:31,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:31,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T22:58:31,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T22:58:31,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:31,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:31,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T22:58:31,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T22:58:31,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:31,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:31,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T22:58:31,414 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T22:58:31,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:31,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:31,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T22:58:31,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740 2024-12-11T22:58:31,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740 2024-12-11T22:58:31,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T22:58:31,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T22:58:31,421 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:58:31,424 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T22:58:31,428 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:31,429 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59670019, jitterRate=-0.11084742844104767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:58:31,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733957911398Initializing all the Stores at 1733957911399 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957911399Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957911399Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957911399Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957911399Cleaning up temporary data from old regions at 1733957911420 (+21 ms)Region opened successfully at 1733957911432 (+12 ms) 2024-12-11T22:58:31,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T22:58:31,432 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T22:58:31,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T22:58:31,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T22:58:31,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T22:58:31,434 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T22:58:31,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733957911432Disabling compacts and flushes for region at 1733957911432Disabling writes for close at 1733957911432Writing region close event to WAL at 1733957911433 (+1 ms)Closed at 1733957911434 (+1 ms) 2024-12-11T22:58:31,437 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T22:58:31,438 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T22:58:31,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T22:58:31,454 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:58:31,458 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T22:58:31,480 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(2659): reportForDuty to master=b48762765ec4,38869,1733957909474 with port=34571, startcode=1733957910258 2024-12-11T22:58:31,480 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(2659): reportForDuty to master=b48762765ec4,38869,1733957909474 with port=37687, startcode=1733957910157 2024-12-11T22:58:31,480 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(2659): reportForDuty to master=b48762765ec4,38869,1733957909474 with port=33507, startcode=1733957910297 2024-12-11T22:58:31,482 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b48762765ec4,37687,1733957910157 2024-12-11T22:58:31,485 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] master.ServerManager(517): Registering regionserver=b48762765ec4,37687,1733957910157 2024-12-11T22:58:31,492 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b48762765ec4,34571,1733957910258 2024-12-11T22:58:31,492 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] master.ServerManager(517): Registering regionserver=b48762765ec4,34571,1733957910258 2024-12-11T22:58:31,493 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c 2024-12-11T22:58:31,493 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38049 2024-12-11T22:58:31,493 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T22:58:31,495 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b48762765ec4,33507,1733957910297 2024-12-11T22:58:31,495 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38869 {}] master.ServerManager(517): Registering regionserver=b48762765ec4,33507,1733957910297 2024-12-11T22:58:31,495 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c 2024-12-11T22:58:31,495 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38049 2024-12-11T22:58:31,495 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T22:58:31,497 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c 2024-12-11T22:58:31,497 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38049 2024-12-11T22:58:31,497 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T22:58:31,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T22:58:31,519 DEBUG [RS:0;b48762765ec4:37687 {}] zookeeper.ZKUtil(111): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b48762765ec4,37687,1733957910157 2024-12-11T22:58:31,519 WARN [RS:0;b48762765ec4:37687 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:58:31,519 INFO [RS:0;b48762765ec4:37687 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:31,519 DEBUG [RS:1;b48762765ec4:34571 {}] zookeeper.ZKUtil(111): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b48762765ec4,34571,1733957910258 2024-12-11T22:58:31,519 WARN [RS:1;b48762765ec4:34571 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:58:31,519 DEBUG [RS:2;b48762765ec4:33507 {}] zookeeper.ZKUtil(111): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b48762765ec4,33507,1733957910297 2024-12-11T22:58:31,519 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157 2024-12-11T22:58:31,519 INFO [RS:1;b48762765ec4:34571 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:31,520 WARN [RS:2;b48762765ec4:33507 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T22:58:31,520 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,34571,1733957910258 2024-12-11T22:58:31,520 INFO [RS:2;b48762765ec4:33507 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:31,520 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,33507,1733957910297 2024-12-11T22:58:31,521 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b48762765ec4,37687,1733957910157] 2024-12-11T22:58:31,521 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b48762765ec4,33507,1733957910297] 2024-12-11T22:58:31,521 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b48762765ec4,34571,1733957910258] 2024-12-11T22:58:31,548 INFO [RS:2;b48762765ec4:33507 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T22:58:31,548 INFO [RS:0;b48762765ec4:37687 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T22:58:31,548 INFO [RS:1;b48762765ec4:34571 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T22:58:31,564 INFO [RS:2;b48762765ec4:33507 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T22:58:31,564 INFO [RS:1;b48762765ec4:34571 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T22:58:31,564 INFO [RS:0;b48762765ec4:37687 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T22:58:31,571 INFO [RS:0;b48762765ec4:37687 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:58:31,571 INFO [RS:1;b48762765ec4:34571 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:58:31,571 INFO [RS:2;b48762765ec4:33507 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:58:31,571 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,571 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,571 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,573 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T22:58:31,573 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T22:58:31,573 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T22:58:31,579 INFO [RS:1;b48762765ec4:34571 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T22:58:31,579 INFO [RS:0;b48762765ec4:37687 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T22:58:31,579 INFO [RS:2;b48762765ec4:33507 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T22:58:31,580 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,580 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,580 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,581 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,581 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:58:31,581 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:58:31,582 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0, corePoolSize=2, maxPoolSize=2 2024-12-11T22:58:31,582 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,582 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b48762765ec4:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:58:31,582 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,583 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,583 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b48762765ec4:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:58:31,583 DEBUG [RS:0;b48762765ec4:37687 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b48762765ec4:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:58:31,583 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b48762765ec4:0, corePoolSize=1, maxPoolSize=1 2024-12-11T22:58:31,583 DEBUG [RS:2;b48762765ec4:33507 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b48762765ec4:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:58:31,583 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b48762765ec4:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:58:31,583 DEBUG [RS:1;b48762765ec4:34571 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b48762765ec4:0, corePoolSize=3, maxPoolSize=3 2024-12-11T22:58:31,585 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,37687,1733957910157-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,585 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,33507,1733957910297-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:58:31,586 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,34571,1733957910258-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:58:31,609 WARN [b48762765ec4:38869 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T22:58:31,610 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T22:58:31,610 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T22:58:31,610 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T22:58:31,612 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,33507,1733957910297-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,612 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,34571,1733957910258-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,612 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,37687,1733957910157-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,612 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,612 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,612 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,613 INFO [RS:1;b48762765ec4:34571 {}] regionserver.Replication(171): b48762765ec4,34571,1733957910258 started 2024-12-11T22:58:31,613 INFO [RS:2;b48762765ec4:33507 {}] regionserver.Replication(171): b48762765ec4,33507,1733957910297 started 2024-12-11T22:58:31,613 INFO [RS:0;b48762765ec4:37687 {}] regionserver.Replication(171): b48762765ec4,37687,1733957910157 started 2024-12-11T22:58:31,631 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,632 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1482): Serving as b48762765ec4,33507,1733957910297, RpcServer on b48762765ec4/172.17.0.3:33507, sessionid=0x100cb900b300003 2024-12-11T22:58:31,633 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T22:58:31,633 DEBUG [RS:2;b48762765ec4:33507 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b48762765ec4,33507,1733957910297 2024-12-11T22:58:31,633 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b48762765ec4,33507,1733957910297' 2024-12-11T22:58:31,633 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T22:58:31,634 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T22:58:31,635 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T22:58:31,635 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T22:58:31,635 DEBUG [RS:2;b48762765ec4:33507 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b48762765ec4,33507,1733957910297 2024-12-11T22:58:31,635 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b48762765ec4,33507,1733957910297' 2024-12-11T22:58:31,635 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T22:58:31,636 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T22:58:31,637 DEBUG [RS:2;b48762765ec4:33507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T22:58:31,637 INFO [RS:2;b48762765ec4:33507 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T22:58:31,637 INFO [RS:2;b48762765ec4:33507 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T22:58:31,640 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,640 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:31,640 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1482): Serving as b48762765ec4,37687,1733957910157, RpcServer on b48762765ec4/172.17.0.3:37687, sessionid=0x100cb900b300001 2024-12-11T22:58:31,640 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1482): Serving as b48762765ec4,34571,1733957910258, RpcServer on b48762765ec4/172.17.0.3:34571, sessionid=0x100cb900b300002 2024-12-11T22:58:31,640 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T22:58:31,640 DEBUG [RS:0;b48762765ec4:37687 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b48762765ec4,37687,1733957910157 2024-12-11T22:58:31,640 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T22:58:31,640 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b48762765ec4,37687,1733957910157' 2024-12-11T22:58:31,640 DEBUG [RS:1;b48762765ec4:34571 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b48762765ec4,34571,1733957910258 2024-12-11T22:58:31,641 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T22:58:31,641 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b48762765ec4,34571,1733957910258' 2024-12-11T22:58:31,641 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T22:58:31,641 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T22:58:31,641 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T22:58:31,642 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T22:58:31,642 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T22:58:31,642 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T22:58:31,642 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T22:58:31,642 DEBUG [RS:0;b48762765ec4:37687 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b48762765ec4,37687,1733957910157 2024-12-11T22:58:31,642 DEBUG [RS:1;b48762765ec4:34571 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b48762765ec4,34571,1733957910258 2024-12-11T22:58:31,642 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b48762765ec4,37687,1733957910157' 2024-12-11T22:58:31,642 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b48762765ec4,34571,1733957910258' 2024-12-11T22:58:31,642 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T22:58:31,642 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T22:58:31,643 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T22:58:31,643 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T22:58:31,644 DEBUG [RS:0;b48762765ec4:37687 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T22:58:31,644 INFO [RS:0;b48762765ec4:37687 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T22:58:31,644 INFO [RS:0;b48762765ec4:37687 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T22:58:31,644 DEBUG [RS:1;b48762765ec4:34571 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T22:58:31,644 INFO [RS:1;b48762765ec4:34571 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T22:58:31,644 INFO [RS:1;b48762765ec4:34571 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T22:58:31,742 INFO [RS:2;b48762765ec4:33507 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:31,745 INFO [RS:0;b48762765ec4:37687 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:31,745 INFO [RS:1;b48762765ec4:34571 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:31,746 INFO [RS:2;b48762765ec4:33507 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b48762765ec4%2C33507%2C1733957910297, suffix=, logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,33507,1733957910297, archiveDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs, maxLogs=32 2024-12-11T22:58:31,748 INFO [RS:0;b48762765ec4:37687 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b48762765ec4%2C37687%2C1733957910157, suffix=, logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157, archiveDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs, maxLogs=32 2024-12-11T22:58:31,748 INFO [RS:1;b48762765ec4:34571 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b48762765ec4%2C34571%2C1733957910258, suffix=, logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,34571,1733957910258, archiveDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs, maxLogs=32 2024-12-11T22:58:31,763 DEBUG [RS:2;b48762765ec4:33507 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,33507,1733957910297/b48762765ec4%2C33507%2C1733957910297.1733957911748, exclude list is [], retry=0 2024-12-11T22:58:31,764 DEBUG [RS:1;b48762765ec4:34571 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,34571,1733957910258/b48762765ec4%2C34571%2C1733957910258.1733957911750, exclude list is [], retry=0 2024-12-11T22:58:31,766 DEBUG [RS:0;b48762765ec4:37687 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157/b48762765ec4%2C37687%2C1733957910157.1733957911750, exclude list is [], retry=0 2024-12-11T22:58:31,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:31,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:31,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:31,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:31,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:31,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:31,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:31,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:31,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:31,777 INFO [RS:2;b48762765ec4:33507 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,33507,1733957910297/b48762765ec4%2C33507%2C1733957910297.1733957911748 2024-12-11T22:58:31,778 DEBUG [RS:2;b48762765ec4:33507 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:31,778 INFO [RS:0;b48762765ec4:37687 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157/b48762765ec4%2C37687%2C1733957910157.1733957911750 2024-12-11T22:58:31,779 DEBUG [RS:0;b48762765ec4:37687 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:31,780 INFO [RS:1;b48762765ec4:34571 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,34571,1733957910258/b48762765ec4%2C34571%2C1733957910258.1733957911750 2024-12-11T22:58:31,780 DEBUG [RS:1;b48762765ec4:34571 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:31,862 DEBUG [b48762765ec4:38869 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-11T22:58:31,873 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(204): Hosts are {b48762765ec4=0} racks are {/default-rack=0} 2024-12-11T22:58:31,881 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:58:31,881 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:58:31,881 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-11T22:58:31,881 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:58:31,881 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:58:31,882 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-11T22:58:31,882 INFO [b48762765ec4:38869 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:58:31,882 INFO [b48762765ec4:38869 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:58:31,882 INFO [b48762765ec4:38869 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-11T22:58:31,882 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:58:31,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b48762765ec4,37687,1733957910157 2024-12-11T22:58:31,903 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b48762765ec4,37687,1733957910157, state=OPENING 2024-12-11T22:58:31,908 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T22:58:31,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:58:31,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:31,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:31,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:31,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:31,913 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:58:31,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b48762765ec4,37687,1733957910157}] 2024-12-11T22:58:32,092 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:58:32,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34123, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:58:32,107 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T22:58:32,108 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:32,109 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T22:58:32,112 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b48762765ec4%2C37687%2C1733957910157.meta, suffix=.meta, logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157, archiveDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs, maxLogs=32 2024-12-11T22:58:32,128 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, exclude list is [], retry=0 2024-12-11T22:58:32,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:32,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:32,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:32,135 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta 2024-12-11T22:58:32,136 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:32,136 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:32,138 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T22:58:32,140 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T22:58:32,145 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T22:58:32,150 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T22:58:32,150 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:32,150 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T22:58:32,150 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T22:58:32,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T22:58:32,155 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T22:58:32,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:32,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:32,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T22:58:32,157 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T22:58:32,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:32,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:32,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T22:58:32,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T22:58:32,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:32,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:32,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T22:58:32,162 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T22:58:32,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:32,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:32,163 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T22:58:32,165 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740 2024-12-11T22:58:32,167 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740 2024-12-11T22:58:32,170 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T22:58:32,170 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T22:58:32,171 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:58:32,174 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T22:58:32,176 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63632449, jitterRate=-0.0518026202917099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:58:32,177 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T22:58:32,179 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733957912151Writing region info on filesystem at 1733957912151Initializing all the Stores at 1733957912153 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957912153Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957912154 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957912154Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957912154Cleaning up temporary data from old regions at 1733957912170 (+16 ms)Running coprocessor post-open hooks at 1733957912177 (+7 ms)Region opened successfully at 1733957912178 (+1 ms) 2024-12-11T22:58:32,186 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733957912083 2024-12-11T22:58:32,197 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T22:58:32,198 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T22:58:32,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b48762765ec4,37687,1733957910157 2024-12-11T22:58:32,203 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b48762765ec4,37687,1733957910157, state=OPEN 2024-12-11T22:58:32,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:32,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:32,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:32,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:32,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:32,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:32,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:32,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:32,206 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b48762765ec4,37687,1733957910157 2024-12-11T22:58:32,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T22:58:32,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b48762765ec4,37687,1733957910157 in 292 msec 2024-12-11T22:58:32,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T22:58:32,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 771 msec 2024-12-11T22:58:32,224 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T22:58:32,224 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T22:58:32,246 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:58:32,247 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1] 2024-12-11T22:58:32,269 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:58:32,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48525, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:58:32,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0710 sec 2024-12-11T22:58:32,290 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733957912290, completionTime=-1 2024-12-11T22:58:32,293 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-11T22:58:32,293 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T22:58:32,325 INFO [master/b48762765ec4:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-11T22:58:32,325 INFO [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733957972325 2024-12-11T22:58:32,325 INFO [master/b48762765ec4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733958032325 2024-12-11T22:58:32,325 INFO [master/b48762765ec4:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-12-11T22:58:32,327 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-11T22:58:32,334 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,38869,1733957909474-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:32,334 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,38869,1733957909474-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:32,334 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,38869,1733957909474-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:32,336 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b48762765ec4:38869, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:32,336 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:32,337 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:32,344 DEBUG [master/b48762765ec4:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T22:58:32,367 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.985sec 2024-12-11T22:58:32,395 INFO [master/b48762765ec4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T22:58:32,397 INFO [master/b48762765ec4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T22:58:32,398 INFO [master/b48762765ec4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T22:58:32,399 INFO [master/b48762765ec4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T22:58:32,399 INFO [master/b48762765ec4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T22:58:32,400 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,38869,1733957909474-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T22:58:32,401 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,38869,1733957909474-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T22:58:32,409 DEBUG [master/b48762765ec4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T22:58:32,410 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T22:58:32,411 INFO [master/b48762765ec4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b48762765ec4,38869,1733957909474-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:32,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba11e32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T22:58:32,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b48762765ec4,38869,-1 for getting cluster id 2024-12-11T22:58:32,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T22:58:32,453 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '55e41239-6683-4c79-8b12-6cb06edb1ab3' 2024-12-11T22:58:32,455 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T22:58:32,455 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "55e41239-6683-4c79-8b12-6cb06edb1ab3" 2024-12-11T22:58:32,456 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b918358, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T22:58:32,456 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b48762765ec4,38869,-1] 2024-12-11T22:58:32,458 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T22:58:32,460 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:58:32,461 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T22:58:32,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5086f397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T22:58:32,464 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:58:32,471 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1] 2024-12-11T22:58:32,471 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:58:32,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:58:32,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b48762765ec4,38869,1733957909474 2024-12-11T22:58:32,494 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:38049/hbase 2024-12-11T22:58:32,509 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=612, MaxFileDescriptor=1048576, SystemLoadAverage=64, ProcessCount=11, AvailableMemoryMB=11890 2024-12-11T22:58:32,528 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:32,532 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:32,534 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:32,538 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53399681, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-53399681, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:32,553 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53399681/hregion-53399681.1733957912540, exclude list is [], retry=0 2024-12-11T22:58:32,558 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:32,558 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:32,558 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:32,562 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53399681/hregion-53399681.1733957912540 2024-12-11T22:58:32,564 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:32,564 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1305f8dbbb1738c71d337feab92b4d55, NAME => 'testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:32,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741838_1014 (size=64) 2024-12-11T22:58:32,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741838_1014 (size=64) 2024-12-11T22:58:32,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741838_1014 (size=64) 2024-12-11T22:58:32,580 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:32,585 INFO [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,588 INFO [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1305f8dbbb1738c71d337feab92b4d55 columnFamilyName a 2024-12-11T22:58:32,588 DEBUG [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:32,590 INFO [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] regionserver.HStore(327): Store=1305f8dbbb1738c71d337feab92b4d55/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:32,590 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,592 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,593 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,595 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,595 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,598 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,602 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:32,603 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1305f8dbbb1738c71d337feab92b4d55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58730279, jitterRate=-0.12485064566135406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:32,605 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1305f8dbbb1738c71d337feab92b4d55: Writing region info on filesystem at 1733957912580Initializing all the Stores at 1733957912582 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957912582Cleaning up temporary data from old regions at 1733957912595 (+13 ms)Region opened successfully at 1733957912605 (+10 ms) 2024-12-11T22:58:32,605 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1305f8dbbb1738c71d337feab92b4d55, disabling compactions & flushes 2024-12-11T22:58:32,605 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55. 2024-12-11T22:58:32,605 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55. 2024-12-11T22:58:32,605 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55. after waiting 0 ms 2024-12-11T22:58:32,606 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55. 2024-12-11T22:58:32,606 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55. 2024-12-11T22:58:32,606 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1305f8dbbb1738c71d337feab92b4d55: Waiting for close lock at 1733957912605Disabling compacts and flushes for region at 1733957912605Disabling writes for close at 1733957912606 (+1 ms)Writing region close event to WAL at 1733957912606Closed at 1733957912606 2024-12-11T22:58:32,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741837_1013 (size=93) 2024-12-11T22:58:32,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741837_1013 (size=93) 2024-12-11T22:58:32,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741837_1013 (size=93) 2024-12-11T22:58:32,619 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:32,619 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53399681:(num 1733957912540) 2024-12-11T22:58:32,621 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-11T22:58:32,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741839_1015 (size=369) 2024-12-11T22:58:32,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741839_1015 (size=369) 2024-12-11T22:58:32,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741839_1015 (size=369) 2024-12-11T22:58:32,642 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-11T22:58:32,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741840_1016 (size=231) 2024-12-11T22:58:32,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741840_1016 (size=231) 2024-12-11T22:58:32,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741840_1016 (size=231) 2024-12-11T22:58:32,679 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1, size=369 (369bytes) 2024-12-11T22:58:32,680 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-11T22:58:32,680 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-11T22:58:32,680 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1 2024-12-11T22:58:32,686 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1 after 4ms 2024-12-11T22:58:32,692 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1 took 14ms 2024-12-11T22:58:32,695 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1 so closing down 2024-12-11T22:58:32,699 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-11T22:58:32,700 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:32,701 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000001-wal-1.temp 2024-12-11T22:58:32,702 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:32,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741841_1017 (size=369) 2024-12-11T22:58:32,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741841_1017 (size=369) 2024-12-11T22:58:32,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741841_1017 (size=369) 2024-12-11T22:58:32,715 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:32,718 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002 2024-12-11T22:58:32,721 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-11T22:58:32,721 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1, journal: Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1, size=369 (369bytes) at 1733957912679Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1 so closing down at 1733957912695 (+16 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000001-wal-1.temp at 1733957912701 (+6 ms)3 split writer threads finished at 1733957912702 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733957912715 (+13 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002 at 1733957912718 (+3 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733957912721 (+3 ms) 2024-12-11T22:58:32,737 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2, size=231 (231bytes) 2024-12-11T22:58:32,737 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2 2024-12-11T22:58:32,737 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2 after 0ms 2024-12-11T22:58:32,746 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2 took 10ms 2024-12-11T22:58:32,749 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2 so closing down 2024-12-11T22:58:32,749 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:32,751 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-11T22:58:32,753 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002-wal-2.temp 2024-12-11T22:58:32,753 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:32,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741842_1018 (size=231) 2024-12-11T22:58:32,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741842_1018 (size=231) 2024-12-11T22:58:32,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741842_1018 (size=231) 2024-12-11T22:58:32,763 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:32,769 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-12-11T22:58:32,771 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-11T22:58:32,771 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2, journal: Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2, size=231 (231bytes) at 1733957912737Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2 so closing down at 1733957912749 (+12 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002-wal-2.temp at 1733957912753 (+4 ms)3 split writer threads finished at 1733957912753Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733957912763 (+10 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733957912771 (+8 ms) 2024-12-11T22:58:32,771 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:32,773 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:32,788 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal.1733957912774, exclude list is [], retry=0 2024-12-11T22:58:32,792 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:32,792 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:32,793 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:32,795 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal.1733957912774 2024-12-11T22:58:32,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:32,796 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1305f8dbbb1738c71d337feab92b4d55, NAME => 'testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:32,796 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:32,796 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,796 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,799 INFO [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,801 INFO [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1305f8dbbb1738c71d337feab92b4d55 columnFamilyName a 2024-12-11T22:58:32,801 DEBUG [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:32,802 INFO [StoreOpener-1305f8dbbb1738c71d337feab92b4d55-1 {}] regionserver.HStore(327): Store=1305f8dbbb1738c71d337feab92b4d55/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:32,802 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,807 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,808 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002 2024-12-11T22:58:32,819 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002 2024-12-11T22:58:32,822 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1305f8dbbb1738c71d337feab92b4d55 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-11T22:58:32,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/.tmp/a/20aa4dbe58ce4c11bf78f960cdb4b083 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733957912619/Put/seqid=0 2024-12-11T22:58:32,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741844_1020 (size=5170) 2024-12-11T22:58:32,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741844_1020 (size=5170) 2024-12-11T22:58:32,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741844_1020 (size=5170) 2024-12-11T22:58:32,890 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/.tmp/a/20aa4dbe58ce4c11bf78f960cdb4b083 2024-12-11T22:58:32,938 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/.tmp/a/20aa4dbe58ce4c11bf78f960cdb4b083 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/a/20aa4dbe58ce4c11bf78f960cdb4b083 2024-12-11T22:58:32,947 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/a/20aa4dbe58ce4c11bf78f960cdb4b083, entries=2, sequenceid=2, filesize=5.0 K 2024-12-11T22:58:32,955 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1305f8dbbb1738c71d337feab92b4d55 in 131ms, sequenceid=2, compaction requested=false; wal=null 2024-12-11T22:58:32,957 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/0000000000000000002 2024-12-11T22:58:32,958 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,958 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,962 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1305f8dbbb1738c71d337feab92b4d55 2024-12-11T22:58:32,966 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/1305f8dbbb1738c71d337feab92b4d55/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-11T22:58:32,967 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1305f8dbbb1738c71d337feab92b4d55; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67104332, jitterRate=-6.753206253051758E-5}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:32,969 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1305f8dbbb1738c71d337feab92b4d55: Writing region info on filesystem at 1733957912797Initializing all the Stores at 1733957912798 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957912799 (+1 ms)Obtaining lock to block concurrent updates at 1733957912822 (+23 ms)Preparing flush snapshotting stores in 1305f8dbbb1738c71d337feab92b4d55 at 1733957912822Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733957912825 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733957912529.1305f8dbbb1738c71d337feab92b4d55. at 1733957912825Flushing 1305f8dbbb1738c71d337feab92b4d55/a: creating writer at 1733957912826 (+1 ms)Flushing 1305f8dbbb1738c71d337feab92b4d55/a: appending metadata at 1733957912866 (+40 ms)Flushing 1305f8dbbb1738c71d337feab92b4d55/a: closing flushed file at 1733957912868 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d5e9e5f: reopening flushed file at 1733957912936 (+68 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1305f8dbbb1738c71d337feab92b4d55 in 131ms, sequenceid=2, compaction requested=false; wal=null at 1733957912955 (+19 ms)Cleaning up temporary data from old regions at 1733957912958 (+3 ms)Region opened successfully at 1733957912968 (+10 ms) 2024-12-11T22:58:32,996 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=375 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:40240 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53472 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53636 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53554 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53412 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38049/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 612) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=64 (was 64), ProcessCount=11 (was 11), AvailableMemoryMB=11879 (was 11890) 2024-12-11T22:58:33,009 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=64, ProcessCount=11, AvailableMemoryMB=11877 2024-12-11T22:58:33,025 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:33,027 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:33,028 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:33,032 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32714847, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-32714847, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:33,046 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32714847/hregion-32714847.1733957913032, exclude list is [], retry=0 2024-12-11T22:58:33,050 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:33,050 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:33,050 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:33,053 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32714847/hregion-32714847.1733957913032 2024-12-11T22:58:33,054 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:33,054 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28861f6a51a24c327a35020a80f20477, NAME => 'testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:33,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741846_1022 (size=64) 2024-12-11T22:58:33,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741846_1022 (size=64) 2024-12-11T22:58:33,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741846_1022 (size=64) 2024-12-11T22:58:33,069 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:33,071 INFO [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,073 INFO [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28861f6a51a24c327a35020a80f20477 columnFamilyName a 2024-12-11T22:58:33,073 DEBUG [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:33,074 INFO [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] regionserver.HStore(327): Store=28861f6a51a24c327a35020a80f20477/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:33,074 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,075 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,076 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,077 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,079 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,083 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:33,083 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28861f6a51a24c327a35020a80f20477; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62204959, jitterRate=-0.07307387888431549}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:33,084 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28861f6a51a24c327a35020a80f20477: Writing region info on filesystem at 1733957913069Initializing all the Stores at 1733957913070 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957913070Cleaning up temporary data from old regions at 1733957913077 (+7 ms)Region opened successfully at 1733957913083 (+6 ms) 2024-12-11T22:58:33,084 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28861f6a51a24c327a35020a80f20477, disabling compactions & flushes 2024-12-11T22:58:33,084 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477. 2024-12-11T22:58:33,084 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477. 2024-12-11T22:58:33,084 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477. after waiting 0 ms 2024-12-11T22:58:33,084 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477. 2024-12-11T22:58:33,084 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477. 2024-12-11T22:58:33,085 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28861f6a51a24c327a35020a80f20477: Waiting for close lock at 1733957913084Disabling compacts and flushes for region at 1733957913084Disabling writes for close at 1733957913084Writing region close event to WAL at 1733957913084Closed at 1733957913084 2024-12-11T22:58:33,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741845_1021 (size=93) 2024-12-11T22:58:33,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741845_1021 (size=93) 2024-12-11T22:58:33,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741845_1021 (size=93) 2024-12-11T22:58:33,092 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:33,092 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32714847:(num 1733957913032) 2024-12-11T22:58:33,093 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-11T22:58:33,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741847_1023 (size=369) 2024-12-11T22:58:33,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741847_1023 (size=369) 2024-12-11T22:58:33,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741847_1023 (size=369) 2024-12-11T22:58:33,106 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-11T22:58:33,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741848_1024 (size=231) 2024-12-11T22:58:33,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741848_1024 (size=231) 2024-12-11T22:58:33,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741848_1024 (size=231) 2024-12-11T22:58:33,134 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2, size=231 (231bytes) 2024-12-11T22:58:33,134 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2 2024-12-11T22:58:33,135 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2 after 1ms 2024-12-11T22:58:33,139 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2 took 5ms 2024-12-11T22:58:33,141 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2 so closing down 2024-12-11T22:58:33,141 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:33,143 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-11T22:58:33,144 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002-wal-2.temp 2024-12-11T22:58:33,145 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:33,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741849_1025 (size=231) 2024-12-11T22:58:33,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741849_1025 (size=231) 2024-12-11T22:58:33,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741849_1025 (size=231) 2024-12-11T22:58:33,153 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:33,155 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002 2024-12-11T22:58:33,155 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-11T22:58:33,156 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2, journal: Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2, size=231 (231bytes) at 1733957913134Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2 so closing down at 1733957913141 (+7 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002-wal-2.temp at 1733957913144 (+3 ms)3 split writer threads finished at 1733957913145 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733957913153 (+8 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002 at 1733957913155 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733957913155 2024-12-11T22:58:33,175 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1, size=369 (369bytes) 2024-12-11T22:58:33,175 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1 2024-12-11T22:58:33,176 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1 after 1ms 2024-12-11T22:58:33,180 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1 took 6ms 2024-12-11T22:58:33,183 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1 so closing down 2024-12-11T22:58:33,183 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:33,185 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-11T22:58:33,187 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000001-wal-1.temp 2024-12-11T22:58:33,187 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:33,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741850_1026 (size=369) 2024-12-11T22:58:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741850_1026 (size=369) 2024-12-11T22:58:33,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741850_1026 (size=369) 2024-12-11T22:58:33,196 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:33,202 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002, length=231 2024-12-11T22:58:33,204 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002 2024-12-11T22:58:33,205 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-11T22:58:33,205 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1, journal: Splitting hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1, size=369 (369bytes) at 1733957913175Finishing writing output for hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1 so closing down at 1733957913183 (+8 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000001-wal-1.temp at 1733957913187 (+4 ms)3 split writer threads finished at 1733957913187Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733957913196 (+9 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002 at 1733957913204 (+8 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733957913205 (+1 ms) 2024-12-11T22:58:33,205 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:33,207 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:33,221 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal.1733957913208, exclude list is [], retry=0 2024-12-11T22:58:33,225 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:33,225 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:33,225 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:33,228 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal.1733957913208 2024-12-11T22:58:33,228 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:33,228 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28861f6a51a24c327a35020a80f20477, NAME => 'testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:33,228 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:33,229 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,229 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,231 INFO [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,233 INFO [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28861f6a51a24c327a35020a80f20477 columnFamilyName a 2024-12-11T22:58:33,233 DEBUG [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:33,234 INFO [StoreOpener-28861f6a51a24c327a35020a80f20477-1 {}] regionserver.HStore(327): Store=28861f6a51a24c327a35020a80f20477/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:33,234 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,238 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002 2024-12-11T22:58:33,242 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002 2024-12-11T22:58:33,242 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28861f6a51a24c327a35020a80f20477 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-11T22:58:33,259 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/.tmp/a/68890034bbe145349a95deae1ce07f5d is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733957913092/Put/seqid=0 2024-12-11T22:58:33,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741852_1028 (size=5170) 2024-12-11T22:58:33,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741852_1028 (size=5170) 2024-12-11T22:58:33,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741852_1028 (size=5170) 2024-12-11T22:58:33,270 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/.tmp/a/68890034bbe145349a95deae1ce07f5d 2024-12-11T22:58:33,278 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/.tmp/a/68890034bbe145349a95deae1ce07f5d as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/a/68890034bbe145349a95deae1ce07f5d 2024-12-11T22:58:33,285 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/a/68890034bbe145349a95deae1ce07f5d, entries=2, sequenceid=2, filesize=5.0 K 2024-12-11T22:58:33,286 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 28861f6a51a24c327a35020a80f20477 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-12-11T22:58:33,287 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/0000000000000000002 2024-12-11T22:58:33,287 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,287 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,289 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28861f6a51a24c327a35020a80f20477 2024-12-11T22:58:33,292 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/28861f6a51a24c327a35020a80f20477/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-11T22:58:33,293 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28861f6a51a24c327a35020a80f20477; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68489180, jitterRate=0.02056831121444702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:33,293 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28861f6a51a24c327a35020a80f20477: Writing region info on filesystem at 1733957913229Initializing all the Stores at 1733957913230 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957913231 (+1 ms)Obtaining lock to block concurrent updates at 1733957913242 (+11 ms)Preparing flush snapshotting stores in 28861f6a51a24c327a35020a80f20477 at 1733957913242Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733957913242Flushing stores of testReplayEditsWrittenIntoWAL,,1733957913026.28861f6a51a24c327a35020a80f20477. at 1733957913242Flushing 28861f6a51a24c327a35020a80f20477/a: creating writer at 1733957913242Flushing 28861f6a51a24c327a35020a80f20477/a: appending metadata at 1733957913258 (+16 ms)Flushing 28861f6a51a24c327a35020a80f20477/a: closing flushed file at 1733957913258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f814053: reopening flushed file at 1733957913277 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 28861f6a51a24c327a35020a80f20477 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1733957913286 (+9 ms)Cleaning up temporary data from old regions at 1733957913287 (+1 ms)Region opened successfully at 1733957913293 (+6 ms) 2024-12-11T22:58:33,310 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53472 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53412 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53712 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53638 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:40160 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:40324 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=64 (was 64), ProcessCount=11 (was 11), AvailableMemoryMB=11873 (was 11877) 2024-12-11T22:58:33,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=64, ProcessCount=11, AvailableMemoryMB=11873 2024-12-11T22:58:33,338 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:33,340 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:33,341 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:33,345 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-94520830, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-94520830, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:33,361 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-94520830/hregion-94520830.1733957913346, exclude list is [], retry=0 2024-12-11T22:58:33,364 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:33,365 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:33,365 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:33,369 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-94520830/hregion-94520830.1733957913346 2024-12-11T22:58:33,370 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:33,370 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 70bf4f245f7f556ccb641fc5887fcb6c, NAME => 'testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:33,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741854_1030 (size=64) 2024-12-11T22:58:33,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741854_1030 (size=64) 2024-12-11T22:58:33,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741854_1030 (size=64) 2024-12-11T22:58:33,382 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:33,384 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,386 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70bf4f245f7f556ccb641fc5887fcb6c columnFamilyName a 2024-12-11T22:58:33,386 DEBUG [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:33,387 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(327): Store=70bf4f245f7f556ccb641fc5887fcb6c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:33,387 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,389 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70bf4f245f7f556ccb641fc5887fcb6c columnFamilyName b 2024-12-11T22:58:33,389 DEBUG [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:33,389 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(327): Store=70bf4f245f7f556ccb641fc5887fcb6c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:33,389 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,391 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70bf4f245f7f556ccb641fc5887fcb6c columnFamilyName c 2024-12-11T22:58:33,391 DEBUG [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:33,392 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(327): Store=70bf4f245f7f556ccb641fc5887fcb6c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:33,392 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,394 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,394 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,395 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:33,397 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:33,400 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:33,400 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 70bf4f245f7f556ccb641fc5887fcb6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63602125, jitterRate=-0.052254483103752136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:33,401 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 70bf4f245f7f556ccb641fc5887fcb6c: Writing region info on filesystem at 1733957913383Initializing all the Stores at 1733957913384 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957913384Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957913384Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957913384Cleaning up temporary data from old regions at 1733957913394 (+10 ms)Region opened successfully at 1733957913400 (+6 ms) 2024-12-11T22:58:33,401 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 70bf4f245f7f556ccb641fc5887fcb6c, disabling compactions & flushes 2024-12-11T22:58:33,401 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:33,401 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:33,401 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. after waiting 0 ms 2024-12-11T22:58:33,401 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:33,401 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:33,401 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 70bf4f245f7f556ccb641fc5887fcb6c: Waiting for close lock at 1733957913401Disabling compacts and flushes for region at 1733957913401Disabling writes for close at 1733957913401Writing region close event to WAL at 1733957913401Closed at 1733957913401 2024-12-11T22:58:33,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741853_1029 (size=93) 2024-12-11T22:58:33,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741853_1029 (size=93) 2024-12-11T22:58:33,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741853_1029 (size=93) 2024-12-11T22:58:33,408 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:33,408 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-94520830:(num 1733957913346) 2024-12-11T22:58:33,408 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:33,410 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:33,424 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411, exclude list is [], retry=0 2024-12-11T22:58:33,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:33,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:33,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:33,430 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 2024-12-11T22:58:33,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:33,626 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411, size=0 (0bytes) 2024-12-11T22:58:33,626 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 might be still open, length is 0 2024-12-11T22:58:33,626 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 2024-12-11T22:58:33,627 WARN [IPC Server handler 4 on default port 38049 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-11T22:58:33,628 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 after 2ms 2024-12-11T22:58:34,407 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:40346 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40346 dst: /127.0.0.1:39291 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39291 remote=/127.0.0.1:40346]. Total timeout mills is 60000, 59186 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:34,408 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53740 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37177:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53740 dst: /127.0.0.1:37177 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:34,408 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:53676 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43249:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53676 dst: /127.0.0.1:43249 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:34,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741855_1032 (size=470675) 2024-12-11T22:58:34,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741855_1032 (size=470675) 2024-12-11T22:58:34,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741855_1032 (size=470675) 2024-12-11T22:58:37,629 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 after 4003ms 2024-12-11T22:58:37,634 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 took 4008ms 2024-12-11T22:58:37,639 INFO [Time-limited test-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733957913411.temp 2024-12-11T22:58:37,642 INFO [Time-limited test-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000000001-wal.1733957913411.temp 2024-12-11T22:58:37,654 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T22:58:37,717 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411; continuing. 2024-12-11T22:58:37,717 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 so closing down 2024-12-11T22:58:37,717 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:37,717 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:37,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741856_1033 (size=470683) 2024-12-11T22:58:37,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741856_1033 (size=470683) 2024-12-11T22:58:37,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741856_1033 (size=470683) 2024-12-11T22:58:37,725 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000000001-wal.1733957913411.temp (wrote 3002 edits, skipped 0 edits in 28 ms) 2024-12-11T22:58:37,727 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000000001-wal.1733957913411.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000003002 2024-12-11T22:58:37,727 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411, size=0, length=0, corrupted=false, cancelled=false 2024-12-11T22:58:37,727 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411, journal: Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411, size=0 (0bytes) at 1733957913626Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000000001-wal.1733957913411.temp at 1733957917642 (+4016 ms)Split 1024 edits, skipped 0 edits. at 1733957917682 (+40 ms)Split 2048 edits, skipped 0 edits. at 1733957917702 (+20 ms)Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 so closing down at 1733957917717 (+15 ms)3 split writer threads finished at 1733957917717Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000000001-wal.1733957913411.temp (wrote 3002 edits, skipped 0 edits in 28 ms) at 1733957917725 (+8 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000000001-wal.1733957913411.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000003002 at 1733957917727 (+2 ms)Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411, size=0, length=0, corrupted=false, cancelled=false at 1733957917727 2024-12-11T22:58:37,729 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957913411 2024-12-11T22:58:37,731 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000003002 2024-12-11T22:58:37,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:37,733 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:37,738 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T22:58:37,747 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957917733, exclude list is [], retry=0 2024-12-11T22:58:37,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:37,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:37,752 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:37,754 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957917733 2024-12-11T22:58:37,754 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:37,754 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:37,756 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:37,757 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70bf4f245f7f556ccb641fc5887fcb6c columnFamilyName a 2024-12-11T22:58:37,757 DEBUG [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:37,758 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(327): Store=70bf4f245f7f556ccb641fc5887fcb6c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:37,758 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:37,759 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70bf4f245f7f556ccb641fc5887fcb6c columnFamilyName b 2024-12-11T22:58:37,759 DEBUG [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:37,759 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(327): Store=70bf4f245f7f556ccb641fc5887fcb6c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:37,760 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:37,761 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 70bf4f245f7f556ccb641fc5887fcb6c columnFamilyName c 2024-12-11T22:58:37,761 DEBUG [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:37,761 INFO [StoreOpener-70bf4f245f7f556ccb641fc5887fcb6c-1 {}] regionserver.HStore(327): Store=70bf4f245f7f556ccb641fc5887fcb6c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:37,761 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:37,762 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:37,764 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:37,765 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000003002 2024-12-11T22:58:37,808 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-11T22:58:38,195 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 70bf4f245f7f556ccb641fc5887fcb6c 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-11T22:58:38,235 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/a/4c56eed6669e4ea8be9a0e6c464ecf64 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733957913441/Put/seqid=0 2024-12-11T22:58:38,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741858_1035 (size=50463) 2024-12-11T22:58:38,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741858_1035 (size=50463) 2024-12-11T22:58:38,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741858_1035 (size=50463) 2024-12-11T22:58:38,246 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/a/4c56eed6669e4ea8be9a0e6c464ecf64 2024-12-11T22:58:38,255 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/a/4c56eed6669e4ea8be9a0e6c464ecf64 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/a/4c56eed6669e4ea8be9a0e6c464ecf64 2024-12-11T22:58:38,262 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/a/4c56eed6669e4ea8be9a0e6c464ecf64, entries=754, sequenceid=754, filesize=49.3 K 2024-12-11T22:58:38,262 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 70bf4f245f7f556ccb641fc5887fcb6c in 67ms, sequenceid=754, compaction requested=false; wal=null 2024-12-11T22:58:38,274 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-11T22:58:38,275 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 70bf4f245f7f556ccb641fc5887fcb6c 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-11T22:58:38,285 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/a/669f21c9142c40518052858edfd83e6c is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733957913487/Put/seqid=0 2024-12-11T22:58:38,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741859_1036 (size=20072) 2024-12-11T22:58:38,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741859_1036 (size=20072) 2024-12-11T22:58:38,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741859_1036 (size=20072) 2024-12-11T22:58:38,296 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/a/669f21c9142c40518052858edfd83e6c 2024-12-11T22:58:38,326 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/b/c06a6547c12b4c65b4979f17190d308f is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733957913508/Put/seqid=0 2024-12-11T22:58:38,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741860_1037 (size=35835) 2024-12-11T22:58:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741860_1037 (size=35835) 2024-12-11T22:58:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741860_1037 (size=35835) 2024-12-11T22:58:38,336 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/b/c06a6547c12b4c65b4979f17190d308f 2024-12-11T22:58:38,343 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/a/669f21c9142c40518052858edfd83e6c as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/a/669f21c9142c40518052858edfd83e6c 2024-12-11T22:58:38,350 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/a/669f21c9142c40518052858edfd83e6c, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-11T22:58:38,351 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/b/c06a6547c12b4c65b4979f17190d308f as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/b/c06a6547c12b4c65b4979f17190d308f 2024-12-11T22:58:38,357 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/b/c06a6547c12b4c65b4979f17190d308f, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-11T22:58:38,358 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 70bf4f245f7f556ccb641fc5887fcb6c in 84ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-11T22:58:38,367 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-11T22:58:38,368 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 70bf4f245f7f556ccb641fc5887fcb6c 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-11T22:58:38,374 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/b/e1ce7bf813d944078485f8e51d5cfd13 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733957913527/Put/seqid=0 2024-12-11T22:58:38,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741861_1038 (size=35082) 2024-12-11T22:58:38,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741861_1038 (size=35082) 2024-12-11T22:58:38,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741861_1038 (size=35082) 2024-12-11T22:58:38,387 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/b/e1ce7bf813d944078485f8e51d5cfd13 2024-12-11T22:58:38,416 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/c/e371598ad278430d9bd47e4d3e9933dd is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733957913552/Put/seqid=0 2024-12-11T22:58:38,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741862_1039 (size=20825) 2024-12-11T22:58:38,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741862_1039 (size=20825) 2024-12-11T22:58:38,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741862_1039 (size=20825) 2024-12-11T22:58:38,424 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/c/e371598ad278430d9bd47e4d3e9933dd 2024-12-11T22:58:38,432 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/b/e1ce7bf813d944078485f8e51d5cfd13 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/b/e1ce7bf813d944078485f8e51d5cfd13 2024-12-11T22:58:38,438 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/b/e1ce7bf813d944078485f8e51d5cfd13, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-11T22:58:38,439 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/c/e371598ad278430d9bd47e4d3e9933dd as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/c/e371598ad278430d9bd47e4d3e9933dd 2024-12-11T22:58:38,445 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/c/e371598ad278430d9bd47e4d3e9933dd, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-11T22:58:38,445 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 70bf4f245f7f556ccb641fc5887fcb6c in 78ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-11T22:58:38,453 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733957913591/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:38,457 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000003002 2024-12-11T22:58:38,457 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-11T22:58:38,458 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 70bf4f245f7f556ccb641fc5887fcb6c 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-11T22:58:38,468 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/c/7ad143903f384cd19e0ad19324ce54a6 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733957913559/Put/seqid=0 2024-12-11T22:58:38,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741863_1040 (size=50301) 2024-12-11T22:58:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741863_1040 (size=50301) 2024-12-11T22:58:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741863_1040 (size=50301) 2024-12-11T22:58:38,476 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/c/7ad143903f384cd19e0ad19324ce54a6 2024-12-11T22:58:38,483 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ad143903f384cd19e0ad19324ce54a6 2024-12-11T22:58:38,486 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/.tmp/c/7ad143903f384cd19e0ad19324ce54a6 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/c/7ad143903f384cd19e0ad19324ce54a6 2024-12-11T22:58:38,493 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ad143903f384cd19e0ad19324ce54a6 2024-12-11T22:58:38,493 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/c/7ad143903f384cd19e0ad19324ce54a6, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-11T22:58:38,493 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 70bf4f245f7f556ccb641fc5887fcb6c in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-11T22:58:38,494 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/0000000000000003002 2024-12-11T22:58:38,496 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:38,496 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:38,497 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T22:58:38,499 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 70bf4f245f7f556ccb641fc5887fcb6c 2024-12-11T22:58:38,508 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenIntoWAL/70bf4f245f7f556ccb641fc5887fcb6c/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-11T22:58:38,510 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 70bf4f245f7f556ccb641fc5887fcb6c; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60458773, jitterRate=-0.09909407794475555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T22:58:38,510 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 70bf4f245f7f556ccb641fc5887fcb6c: Writing region info on filesystem at 1733957917754Initializing all the Stores at 1733957917756 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957917756Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957917756Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957917756Cleaning up temporary data from old regions at 1733957918496 (+740 ms)Region opened successfully at 1733957918510 (+14 ms) 2024-12-11T22:58:38,570 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 70bf4f245f7f556ccb641fc5887fcb6c, disabling compactions & flushes 2024-12-11T22:58:38,570 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:38,570 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:38,570 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. after waiting 0 ms 2024-12-11T22:58:38,570 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:38,603 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733957913338.70bf4f245f7f556ccb641fc5887fcb6c. 2024-12-11T22:58:38,603 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 70bf4f245f7f556ccb641fc5887fcb6c: Waiting for close lock at 1733957918570Disabling compacts and flushes for region at 1733957918570Disabling writes for close at 1733957918570Writing region close event to WAL at 1733957918603 (+33 ms)Closed at 1733957918603 2024-12-11T22:58:38,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741857_1034 (size=93) 2024-12-11T22:58:38,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741857_1034 (size=93) 2024-12-11T22:58:38,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741857_1034 (size=93) 2024-12-11T22:58:38,611 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:38,611 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733957917733) 2024-12-11T22:58:38,633 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=401 (was 385) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-933926383_22 at /127.0.0.1:37898 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (80400340) connection to localhost/127.0.0.1:43201 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (80400340) connection to localhost/127.0.0.1:38049 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43201 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (80400340) connection to localhost/127.0.0.1:42893 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:38049 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-933926383_22 at /127.0.0.1:49972 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-933926383_22 at /127.0.0.1:37924 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42893 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-933926383_22 at /127.0.0.1:41560 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=59 (was 64), ProcessCount=11 (was 11), AvailableMemoryMB=11825 (was 11873) 2024-12-11T22:58:38,645 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=59, ProcessCount=11, AvailableMemoryMB=11822 2024-12-11T22:58:38,661 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:38,664 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:38,664 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:38,667 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-01182189, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-01182189, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:38,681 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-01182189/hregion-01182189.1733957918667, exclude list is [], retry=0 2024-12-11T22:58:38,684 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:38,684 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:38,685 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:38,687 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-01182189/hregion-01182189.1733957918667 2024-12-11T22:58:38,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:38,688 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5a2592b65df88f7335997617e4d1e807, NAME => 'test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:38,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741865_1042 (size=43) 2024-12-11T22:58:38,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741865_1042 (size=43) 2024-12-11T22:58:38,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741865_1042 (size=43) 2024-12-11T22:58:38,701 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:38,703 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,705 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2592b65df88f7335997617e4d1e807 columnFamilyName a 2024-12-11T22:58:38,705 DEBUG [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:38,705 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(327): Store=5a2592b65df88f7335997617e4d1e807/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:38,705 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,707 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2592b65df88f7335997617e4d1e807 columnFamilyName b 2024-12-11T22:58:38,707 DEBUG [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:38,708 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(327): Store=5a2592b65df88f7335997617e4d1e807/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:38,708 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,710 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2592b65df88f7335997617e4d1e807 columnFamilyName c 2024-12-11T22:58:38,710 DEBUG [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:38,710 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(327): Store=5a2592b65df88f7335997617e4d1e807/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:38,710 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,713 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,713 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,714 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:38,715 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:38,718 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:38,718 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a2592b65df88f7335997617e4d1e807; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63801189, jitterRate=-0.049288198351860046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:38,720 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a2592b65df88f7335997617e4d1e807: Writing region info on filesystem at 1733957918702Initializing all the Stores at 1733957918703 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957918703Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957918703Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957918703Cleaning up temporary data from old regions at 1733957918713 (+10 ms)Region opened successfully at 1733957918719 (+6 ms) 2024-12-11T22:58:38,720 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5a2592b65df88f7335997617e4d1e807, disabling compactions & flushes 2024-12-11T22:58:38,720 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:38,720 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:38,720 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. after waiting 0 ms 2024-12-11T22:58:38,720 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:38,720 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:38,721 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5a2592b65df88f7335997617e4d1e807: Waiting for close lock at 1733957918720Disabling compacts and flushes for region at 1733957918720Disabling writes for close at 1733957918720Writing region close event to WAL at 1733957918720Closed at 1733957918720 2024-12-11T22:58:38,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741864_1041 (size=93) 2024-12-11T22:58:38,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741864_1041 (size=93) 2024-12-11T22:58:38,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741864_1041 (size=93) 2024-12-11T22:58:38,729 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:38,729 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-01182189:(num 1733957918667) 2024-12-11T22:58:38,730 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:38,732 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:38,746 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732, exclude list is [], retry=0 2024-12-11T22:58:38,749 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:38,750 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:38,750 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:38,753 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732 2024-12-11T22:58:38,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741866_1043 (size=344306) 2024-12-11T22:58:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741866_1043 (size=344306) 2024-12-11T22:58:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741866_1043 (size=344306) 2024-12-11T22:58:38,960 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732, size=336.2 K (344306bytes) 2024-12-11T22:58:38,960 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732 2024-12-11T22:58:38,961 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732 after 1ms 2024-12-11T22:58:38,965 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732 took 5ms 2024-12-11T22:58:38,970 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733957918732.temp 2024-12-11T22:58:38,972 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000000001-wal.1733957918732.temp 2024-12-11T22:58:39,008 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732 so closing down 2024-12-11T22:58:39,008 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:39,009 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:39,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741867_1044 (size=344306) 2024-12-11T22:58:39,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741867_1044 (size=344306) 2024-12-11T22:58:39,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741867_1044 (size=344306) 2024-12-11T22:58:39,014 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000000001-wal.1733957918732.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-12-11T22:58:39,016 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000000001-wal.1733957918732.temp to hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003000 2024-12-11T22:58:39,016 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-12-11T22:58:39,017 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732, journal: Splitting hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732, size=336.2 K (344306bytes) at 1733957918960Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000000001-wal.1733957918732.temp at 1733957918972 (+12 ms)Split 1024 edits, skipped 0 edits. at 1733957918981 (+9 ms)Split 2048 edits, skipped 0 edits. at 1733957918995 (+14 ms)Finishing writing output for hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732 so closing down at 1733957919008 (+13 ms)3 split writer threads finished at 1733957919009 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000000001-wal.1733957918732.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1733957919014 (+5 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000000001-wal.1733957918732.temp to hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003000 at 1733957919016 (+2 ms)Processed 3000 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1733957919016 2024-12-11T22:58:39,019 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957918732 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957918732 2024-12-11T22:58:39,020 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003000 2024-12-11T22:58:39,020 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:39,022 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:39,037 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022, exclude list is [], retry=0 2024-12-11T22:58:39,041 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:39,041 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:39,042 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:39,044 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022 2024-12-11T22:58:39,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:39,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741868_1045 (size=344433) 2024-12-11T22:58:39,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741868_1045 (size=344433) 2024-12-11T22:58:39,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741868_1045 (size=344433) 2024-12-11T22:58:39,226 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022, size=336.4 K (344433bytes) 2024-12-11T22:58:39,226 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022 2024-12-11T22:58:39,227 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022 after 1ms 2024-12-11T22:58:39,231 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022 took 5ms 2024-12-11T22:58:39,235 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733957919022.temp 2024-12-11T22:58:39,237 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003001-wal.1733957919022.temp 2024-12-11T22:58:39,264 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022 so closing down 2024-12-11T22:58:39,265 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:39,265 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:39,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741869_1046 (size=344433) 2024-12-11T22:58:39,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741869_1046 (size=344433) 2024-12-11T22:58:39,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741869_1046 (size=344433) 2024-12-11T22:58:39,269 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003001-wal.1733957919022.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-12-11T22:58:39,271 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003001-wal.1733957919022.temp to hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000006000 2024-12-11T22:58:39,272 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-12-11T22:58:39,272 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022, journal: Splitting hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022, size=336.4 K (344433bytes) at 1733957919226Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003001-wal.1733957919022.temp at 1733957919237 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733957919243 (+6 ms)Split 2048 edits, skipped 0 edits. at 1733957919254 (+11 ms)Finishing writing output for hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022 so closing down at 1733957919264 (+10 ms)3 split writer threads finished at 1733957919265 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003001-wal.1733957919022.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1733957919269 (+4 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003001-wal.1733957919022.temp to hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000006000 at 1733957919272 (+3 ms)Processed 3000 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1733957919272 2024-12-11T22:58:39,274 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919022 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957919022 2024-12-11T22:58:39,275 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000006000 2024-12-11T22:58:39,276 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:39,278 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/test2727-manual,16010,1733957918660, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:39,302 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919279, exclude list is [], retry=0 2024-12-11T22:58:39,306 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:39,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:39,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:39,310 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733957918660/wal.1733957919279 2024-12-11T22:58:39,311 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:39,311 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a2592b65df88f7335997617e4d1e807, NAME => 'test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:39,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:39,311 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,311 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,313 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,315 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2592b65df88f7335997617e4d1e807 columnFamilyName a 2024-12-11T22:58:39,315 DEBUG [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:39,315 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(327): Store=5a2592b65df88f7335997617e4d1e807/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:39,315 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,316 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2592b65df88f7335997617e4d1e807 columnFamilyName b 2024-12-11T22:58:39,316 DEBUG [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:39,317 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(327): Store=5a2592b65df88f7335997617e4d1e807/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:39,317 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,318 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a2592b65df88f7335997617e4d1e807 columnFamilyName c 2024-12-11T22:58:39,318 DEBUG [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:39,319 INFO [StoreOpener-5a2592b65df88f7335997617e4d1e807-1 {}] regionserver.HStore(327): Store=5a2592b65df88f7335997617e4d1e807/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:39,319 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,320 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,322 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,323 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003000 2024-12-11T22:58:39,359 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003000 2024-12-11T22:58:39,361 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000006000 2024-12-11T22:58:39,396 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000006000 2024-12-11T22:58:39,396 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5a2592b65df88f7335997617e4d1e807 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-11T22:58:39,420 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/a/26b568c11d844b99854608333e96f61c is 41, key is test2727/a:100/1733957919051/Put/seqid=0 2024-12-11T22:58:39,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741871_1048 (size=84227) 2024-12-11T22:58:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741871_1048 (size=84227) 2024-12-11T22:58:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741871_1048 (size=84227) 2024-12-11T22:58:39,429 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/a/26b568c11d844b99854608333e96f61c 2024-12-11T22:58:39,471 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/b/dac3f0b59ee941158a1d7d59500d76fb is 41, key is test2727/b:100/1733957919113/Put/seqid=0 2024-12-11T22:58:39,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741872_1049 (size=84609) 2024-12-11T22:58:39,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741872_1049 (size=84609) 2024-12-11T22:58:39,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741872_1049 (size=84609) 2024-12-11T22:58:39,478 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/b/dac3f0b59ee941158a1d7d59500d76fb 2024-12-11T22:58:39,515 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/c/527ab088820b4237a0adbfda750588ff is 41, key is test2727/c:100/1733957919159/Put/seqid=0 2024-12-11T22:58:39,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741873_1050 (size=84609) 2024-12-11T22:58:39,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741873_1050 (size=84609) 2024-12-11T22:58:39,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741873_1050 (size=84609) 2024-12-11T22:58:39,522 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/c/527ab088820b4237a0adbfda750588ff 2024-12-11T22:58:39,530 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/a/26b568c11d844b99854608333e96f61c as hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/a/26b568c11d844b99854608333e96f61c 2024-12-11T22:58:39,537 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/a/26b568c11d844b99854608333e96f61c, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-11T22:58:39,539 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/b/dac3f0b59ee941158a1d7d59500d76fb as hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/b/dac3f0b59ee941158a1d7d59500d76fb 2024-12-11T22:58:39,546 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/b/dac3f0b59ee941158a1d7d59500d76fb, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-11T22:58:39,547 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/.tmp/c/527ab088820b4237a0adbfda750588ff as hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/c/527ab088820b4237a0adbfda750588ff 2024-12-11T22:58:39,554 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/c/527ab088820b4237a0adbfda750588ff, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-11T22:58:39,554 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 5a2592b65df88f7335997617e4d1e807 in 158ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-11T22:58:39,555 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000003000 2024-12-11T22:58:39,556 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/0000000000000006000 2024-12-11T22:58:39,557 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,558 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,558 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:39,560 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a2592b65df88f7335997617e4d1e807 2024-12-11T22:58:39,563 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/test2727/5a2592b65df88f7335997617e4d1e807/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-11T22:58:39,564 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a2592b65df88f7335997617e4d1e807; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69836868, jitterRate=0.04065042734146118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:39,566 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a2592b65df88f7335997617e4d1e807: Writing region info on filesystem at 1733957919312Initializing all the Stores at 1733957919313 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919313Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919313Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919313Obtaining lock to block concurrent updates at 1733957919396 (+83 ms)Preparing flush snapshotting stores in 5a2592b65df88f7335997617e4d1e807 at 1733957919396Finished memstore snapshotting test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733957919396Flushing stores of test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. at 1733957919396Flushing 5a2592b65df88f7335997617e4d1e807/a: creating writer at 1733957919396Flushing 5a2592b65df88f7335997617e4d1e807/a: appending metadata at 1733957919419 (+23 ms)Flushing 5a2592b65df88f7335997617e4d1e807/a: closing flushed file at 1733957919420 (+1 ms)Flushing 5a2592b65df88f7335997617e4d1e807/b: creating writer at 1733957919437 (+17 ms)Flushing 5a2592b65df88f7335997617e4d1e807/b: appending metadata at 1733957919468 (+31 ms)Flushing 5a2592b65df88f7335997617e4d1e807/b: closing flushed file at 1733957919469 (+1 ms)Flushing 5a2592b65df88f7335997617e4d1e807/c: creating writer at 1733957919485 (+16 ms)Flushing 5a2592b65df88f7335997617e4d1e807/c: appending metadata at 1733957919513 (+28 ms)Flushing 5a2592b65df88f7335997617e4d1e807/c: closing flushed file at 1733957919513Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c3e285: reopening flushed file at 1733957919529 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fd2d21b: reopening flushed file at 1733957919537 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19dd710a: reopening flushed file at 1733957919546 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 5a2592b65df88f7335997617e4d1e807 in 158ms, sequenceid=6000, compaction requested=false; wal=null at 1733957919555 (+9 ms)Cleaning up temporary data from old regions at 1733957919558 (+3 ms)Region opened successfully at 1733957919566 (+8 ms) 2024-12-11T22:58:39,568 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-11T22:58:39,568 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5a2592b65df88f7335997617e4d1e807, disabling compactions & flushes 2024-12-11T22:58:39,568 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:39,568 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:39,568 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. after waiting 0 ms 2024-12-11T22:58:39,568 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:39,569 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733957918662.5a2592b65df88f7335997617e4d1e807. 2024-12-11T22:58:39,570 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5a2592b65df88f7335997617e4d1e807: Waiting for close lock at 1733957919568Disabling compacts and flushes for region at 1733957919568Disabling writes for close at 1733957919568Writing region close event to WAL at 1733957919569 (+1 ms)Closed at 1733957919569 2024-12-11T22:58:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741870_1047 (size=93) 2024-12-11T22:58:39,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741870_1047 (size=93) 2024-12-11T22:58:39,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741870_1047 (size=93) 2024-12-11T22:58:39,578 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:39,578 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733957919279) 2024-12-11T22:58:39,598 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=403 (was 401) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:41680 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:50102 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:37924 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=119 (was 59) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11638 (was 11822) 2024-12-11T22:58:39,613 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=11637 2024-12-11T22:58:39,639 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:39,647 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:39,648 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733957919648 2024-12-11T22:58:39,656 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648 2024-12-11T22:58:39,658 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:39,659 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d1df618e8ba932b8d82f5864cdb51714, NAME => 'testSequentialEditLogSeqNum,,1733957919640.d1df618e8ba932b8d82f5864cdb51714.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:39,659 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733957919640.d1df618e8ba932b8d82f5864cdb51714.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:39,659 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,660 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,660 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714 doesn't exist for region: d1df618e8ba932b8d82f5864cdb51714 on table testSequentialEditLogSeqNum 2024-12-11T22:58:39,661 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: d1df618e8ba932b8d82f5864cdb51714 on table testSequentialEditLogSeqNum 2024-12-11T22:58:39,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741875_1052 (size=62) 2024-12-11T22:58:39,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741875_1052 (size=62) 2024-12-11T22:58:39,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741875_1052 (size=62) 2024-12-11T22:58:39,672 INFO [StoreOpener-d1df618e8ba932b8d82f5864cdb51714-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,674 INFO [StoreOpener-d1df618e8ba932b8d82f5864cdb51714-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1df618e8ba932b8d82f5864cdb51714 columnFamilyName a 2024-12-11T22:58:39,674 DEBUG [StoreOpener-d1df618e8ba932b8d82f5864cdb51714-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:39,674 INFO [StoreOpener-d1df618e8ba932b8d82f5864cdb51714-1 {}] regionserver.HStore(327): Store=d1df618e8ba932b8d82f5864cdb51714/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:39,675 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,675 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,676 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,676 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,676 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,678 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1df618e8ba932b8d82f5864cdb51714 2024-12-11T22:58:39,680 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:39,681 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1df618e8ba932b8d82f5864cdb51714; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61190787, jitterRate=-0.08818621933460236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:39,682 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1df618e8ba932b8d82f5864cdb51714: Writing region info on filesystem at 1733957919660Initializing all the Stores at 1733957919672 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919672Cleaning up temporary data from old regions at 1733957919676 (+4 ms)Region opened successfully at 1733957919682 (+6 ms) 2024-12-11T22:58:39,697 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d1df618e8ba932b8d82f5864cdb51714 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-11T22:58:39,731 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/.tmp/a/e9432b2848464c8b8ca381e61e0b5b05 is 81, key is testSequentialEditLogSeqNum/a:x0/1733957919682/Put/seqid=0 2024-12-11T22:58:39,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741876_1053 (size=5833) 2024-12-11T22:58:39,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741876_1053 (size=5833) 2024-12-11T22:58:39,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741876_1053 (size=5833) 2024-12-11T22:58:39,741 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/.tmp/a/e9432b2848464c8b8ca381e61e0b5b05 2024-12-11T22:58:39,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/.tmp/a/e9432b2848464c8b8ca381e61e0b5b05 as hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/a/e9432b2848464c8b8ca381e61e0b5b05 2024-12-11T22:58:39,757 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/a/e9432b2848464c8b8ca381e61e0b5b05, entries=10, sequenceid=13, filesize=5.7 K 2024-12-11T22:58:39,758 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d1df618e8ba932b8d82f5864cdb51714 in 61ms, sequenceid=13, compaction requested=false 2024-12-11T22:58:39,758 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d1df618e8ba932b8d82f5864cdb51714: 2024-12-11T22:58:39,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-11T22:58:39,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-11T22:58:39,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-11T22:58:39,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-11T22:58:39,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-11T22:58:39,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741874_1051 (size=3017) 2024-12-11T22:58:39,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741874_1051 (size=3017) 2024-12-11T22:58:39,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741874_1051 (size=3017) 2024-12-11T22:58:39,794 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648, size=2.9 K (3017bytes) 2024-12-11T22:58:39,795 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648 2024-12-11T22:58:39,795 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648 after 0ms 2024-12-11T22:58:39,799 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648 took 5ms 2024-12-11T22:58:39,801 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648 so closing down 2024-12-11T22:58:39,802 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:39,803 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733957919648.temp 2024-12-11T22:58:39,804 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000003-wal.1733957919648.temp 2024-12-11T22:58:39,805 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741877_1054 (size=2433) 2024-12-11T22:58:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741877_1054 (size=2433) 2024-12-11T22:58:39,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741877_1054 (size=2433) 2024-12-11T22:58:39,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000003-wal.1733957919648.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:39,817 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000003-wal.1733957919648.temp to hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000020 2024-12-11T22:58:39,818 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-12-11T22:58:39,818 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648, journal: Splitting hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648, size=2.9 K (3017bytes) at 1733957919795Finishing writing output for hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648 so closing down at 1733957919801 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000003-wal.1733957919648.temp at 1733957919805 (+4 ms)3 split writer threads finished at 1733957919805Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000003-wal.1733957919648.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733957919815 (+10 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000003-wal.1733957919648.temp to hdfs://localhost:38049/hbase/data/default/testSequentialEditLogSeqNum/d1df618e8ba932b8d82f5864cdb51714/recovered.edits/0000000000000000020 at 1733957919818 (+3 ms)Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:38049/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733957919638/wal.1733957919648, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1733957919818 2024-12-11T22:58:39,838 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:41680 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:37924 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=119 (was 119), ProcessCount=11 (was 11), AvailableMemoryMB=11606 (was 11637) 2024-12-11T22:58:39,851 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=11604 2024-12-11T22:58:39,869 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:39,871 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:39,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-11T22:58:39,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:39,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T22:58:39,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-11T22:58:39,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-11T22:58:39,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:39,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-11T22:58:39,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:39,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-11T22:58:39,895 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:39,897 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:39,900 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58708379, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-58708379, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:39,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58708379/hregion-58708379.1733957919901, exclude list is [], retry=0 2024-12-11T22:58:39,920 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:39,920 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:39,920 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:39,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58708379/hregion-58708379.1733957919901 2024-12-11T22:58:39,928 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:39,928 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1ef5020c37ba257a882b0e35cf709fc3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741879_1056 (size=70) 2024-12-11T22:58:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741879_1056 (size=70) 2024-12-11T22:58:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741879_1056 (size=70) 2024-12-11T22:58:39,944 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:39,946 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,948 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName a 2024-12-11T22:58:39,948 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:39,948 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:39,948 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,950 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName b 2024-12-11T22:58:39,950 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:39,950 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:39,951 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,952 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName c 2024-12-11T22:58:39,952 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:39,953 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:39,953 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,954 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,954 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,955 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,955 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,956 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:39,958 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,961 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:39,961 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1ef5020c37ba257a882b0e35cf709fc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63828662, jitterRate=-0.04887881875038147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:39,963 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1ef5020c37ba257a882b0e35cf709fc3: Writing region info on filesystem at 1733957919944Initializing all the Stores at 1733957919945 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919946 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919946Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919946Cleaning up temporary data from old regions at 1733957919955 (+9 ms)Region opened successfully at 1733957919963 (+8 ms) 2024-12-11T22:58:39,964 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1ef5020c37ba257a882b0e35cf709fc3, disabling compactions & flushes 2024-12-11T22:58:39,964 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:39,964 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:39,964 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. after waiting 0 ms 2024-12-11T22:58:39,964 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:39,964 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:39,964 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1ef5020c37ba257a882b0e35cf709fc3: Waiting for close lock at 1733957919963Disabling compacts and flushes for region at 1733957919963Disabling writes for close at 1733957919964 (+1 ms)Writing region close event to WAL at 1733957919964Closed at 1733957919964 2024-12-11T22:58:39,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741878_1055 (size=93) 2024-12-11T22:58:39,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741878_1055 (size=93) 2024-12-11T22:58:39,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741878_1055 (size=93) 2024-12-11T22:58:39,971 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:39,971 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58708379:(num 1733957919901) 2024-12-11T22:58:39,971 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:39,973 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:39,989 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974, exclude list is [], retry=0 2024-12-11T22:58:39,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:39,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:39,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:39,996 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 2024-12-11T22:58:39,996 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:39,996 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ef5020c37ba257a882b0e35cf709fc3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:39,996 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:39,997 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,997 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,998 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:39,999 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName a 2024-12-11T22:58:40,000 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:40,000 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:40,000 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,001 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName b 2024-12-11T22:58:40,001 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:40,002 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:40,002 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,003 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName c 2024-12-11T22:58:40,003 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:40,003 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:40,004 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,004 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,005 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,007 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,007 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,007 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:40,009 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:40,009 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1ef5020c37ba257a882b0e35cf709fc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69739275, jitterRate=0.03919617831707001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:40,011 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1ef5020c37ba257a882b0e35cf709fc3: Writing region info on filesystem at 1733957919997Initializing all the Stores at 1733957919998 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919998Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919998Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957919998Cleaning up temporary data from old regions at 1733957920007 (+9 ms)Region opened successfully at 1733957920010 (+3 ms) 2024-12-11T22:58:40,015 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733957920014/Put/seqid=0 2024-12-11T22:58:40,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741881_1058 (size=4826) 2024-12-11T22:58:40,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741881_1058 (size=4826) 2024-12-11T22:58:40,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741881_1058 (size=4826) 2024-12-11T22:58:40,025 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38049/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 1ef5020c37ba257a882b0e35cf709fc3/a 2024-12-11T22:58:40,033 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-11T22:58:40,034 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-11T22:58:40,034 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1ef5020c37ba257a882b0e35cf709fc3: 2024-12-11T22:58:40,036 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/a/3bb6c0618ef84711a6acef46cca1e7ca_SeqId_3_ 2024-12-11T22:58:40,037 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38049/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1ef5020c37ba257a882b0e35cf709fc3/a as hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/a/3bb6c0618ef84711a6acef46cca1e7ca_SeqId_3_ - updating store file list. 2024-12-11T22:58:40,043 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3bb6c0618ef84711a6acef46cca1e7ca_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-11T22:58:40,043 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/a/3bb6c0618ef84711a6acef46cca1e7ca_SeqId_3_ into 1ef5020c37ba257a882b0e35cf709fc3/a 2024-12-11T22:58:40,043 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38049/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1ef5020c37ba257a882b0e35cf709fc3/a (new location: hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/a/3bb6c0618ef84711a6acef46cca1e7ca_SeqId_3_) 2024-12-11T22:58:40,088 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974, size=0 (0bytes) 2024-12-11T22:58:40,088 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 might be still open, length is 0 2024-12-11T22:58:40,088 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 2024-12-11T22:58:40,089 WARN [IPC Server handler 0 on default port 38049 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-11T22:58:40,089 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 after 1ms 2024-12-11T22:58:40,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:50204 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43249:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50204 dst: /127.0.0.1:43249 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43249 remote=/127.0.0.1:50204]. Total timeout mills is 60000, 59661 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:40,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:41764 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41764 dst: /127.0.0.1:39291 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:40,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:38104 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37177:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38104 dst: /127.0.0.1:37177 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:40,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741880_1059 (size=504) 2024-12-11T22:58:40,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741880_1059 (size=504) 2024-12-11T22:58:44,090 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 after 4002ms 2024-12-11T22:58:44,159 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 took 4071ms 2024-12-11T22:58:44,161 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974; continuing. 2024-12-11T22:58:44,161 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 so closing down 2024-12-11T22:58:44,161 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:44,163 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733957919974.temp 2024-12-11T22:58:44,165 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005-wal.1733957919974.temp 2024-12-11T22:58:44,165 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:44,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741882_1060 (size=238) 2024-12-11T22:58:44,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741882_1060 (size=238) 2024-12-11T22:58:44,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741882_1060 (size=238) 2024-12-11T22:58:44,198 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005-wal.1733957919974.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:44,200 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005-wal.1733957919974.temp to hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005 2024-12-11T22:58:44,200 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974, size=0, length=0, corrupted=false, cancelled=false 2024-12-11T22:58:44,200 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974, journal: Splitting hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974, size=0 (0bytes) at 1733957920088Finishing writing output for hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 so closing down at 1733957924161 (+4073 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005-wal.1733957919974.temp at 1733957924165 (+4 ms)3 split writer threads finished at 1733957924165Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005-wal.1733957919974.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733957924198 (+33 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005-wal.1733957919974.temp to hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005 at 1733957924200 (+2 ms)Processed 2 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974, size=0, length=0, corrupted=false, cancelled=false at 1733957924200 2024-12-11T22:58:44,202 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957919974 2024-12-11T22:58:44,203 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005 2024-12-11T22:58:44,203 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:44,205 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:44,221 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957924206, exclude list is [], retry=0 2024-12-11T22:58:44,223 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:44,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:44,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:44,226 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957924206 2024-12-11T22:58:44,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:44,226 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ef5020c37ba257a882b0e35cf709fc3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:44,227 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:44,227 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,227 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,228 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,229 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName a 2024-12-11T22:58:44,229 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:44,236 DEBUG [StoreFileOpener-1ef5020c37ba257a882b0e35cf709fc3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3bb6c0618ef84711a6acef46cca1e7ca_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-11T22:58:44,236 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/a/3bb6c0618ef84711a6acef46cca1e7ca_SeqId_3_ 2024-12-11T22:58:44,237 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:44,237 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,238 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName b 2024-12-11T22:58:44,238 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:44,238 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:44,238 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,239 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ef5020c37ba257a882b0e35cf709fc3 columnFamilyName c 2024-12-11T22:58:44,239 DEBUG [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:44,240 INFO [StoreOpener-1ef5020c37ba257a882b0e35cf709fc3-1 {}] regionserver.HStore(327): Store=1ef5020c37ba257a882b0e35cf709fc3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:44,240 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,241 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,242 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,243 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005 2024-12-11T22:58:44,246 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005 2024-12-11T22:58:44,246 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1ef5020c37ba257a882b0e35cf709fc3 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-11T22:58:44,292 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/.tmp/a/9c60b590705e4968955cf5cd0d27669d is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733957920048/Put/seqid=0 2024-12-11T22:58:44,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741884_1062 (size=5149) 2024-12-11T22:58:44,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741884_1062 (size=5149) 2024-12-11T22:58:44,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741884_1062 (size=5149) 2024-12-11T22:58:44,299 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/.tmp/a/9c60b590705e4968955cf5cd0d27669d 2024-12-11T22:58:44,305 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/.tmp/a/9c60b590705e4968955cf5cd0d27669d as hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/a/9c60b590705e4968955cf5cd0d27669d 2024-12-11T22:58:44,311 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/a/9c60b590705e4968955cf5cd0d27669d, entries=1, sequenceid=5, filesize=5.0 K 2024-12-11T22:58:44,311 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1ef5020c37ba257a882b0e35cf709fc3 in 65ms, sequenceid=5, compaction requested=false; wal=null 2024-12-11T22:58:44,312 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/0000000000000000005 2024-12-11T22:58:44,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,313 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,314 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:44,315 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1ef5020c37ba257a882b0e35cf709fc3 2024-12-11T22:58:44,317 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1ef5020c37ba257a882b0e35cf709fc3/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-11T22:58:44,318 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1ef5020c37ba257a882b0e35cf709fc3; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72309473, jitterRate=0.07749511301517487}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:44,318 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1ef5020c37ba257a882b0e35cf709fc3: Writing region info on filesystem at 1733957924227Initializing all the Stores at 1733957924228 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957924228Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957924228Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957924228Obtaining lock to block concurrent updates at 1733957924246 (+18 ms)Preparing flush snapshotting stores in 1ef5020c37ba257a882b0e35cf709fc3 at 1733957924246Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733957924246Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. at 1733957924246Flushing 1ef5020c37ba257a882b0e35cf709fc3/a: creating writer at 1733957924246Flushing 1ef5020c37ba257a882b0e35cf709fc3/a: appending metadata at 1733957924291 (+45 ms)Flushing 1ef5020c37ba257a882b0e35cf709fc3/a: closing flushed file at 1733957924291Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40301c9e: reopening flushed file at 1733957924305 (+14 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1ef5020c37ba257a882b0e35cf709fc3 in 65ms, sequenceid=5, compaction requested=false; wal=null at 1733957924311 (+6 ms)Cleaning up temporary data from old regions at 1733957924313 (+2 ms)Region opened successfully at 1733957924318 (+5 ms) 2024-12-11T22:58:44,322 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1ef5020c37ba257a882b0e35cf709fc3, disabling compactions & flushes 2024-12-11T22:58:44,322 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:44,322 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:44,323 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. after waiting 1 ms 2024-12-11T22:58:44,323 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:44,323 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733957919869.1ef5020c37ba257a882b0e35cf709fc3. 2024-12-11T22:58:44,323 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1ef5020c37ba257a882b0e35cf709fc3: Waiting for close lock at 1733957924322Disabling compacts and flushes for region at 1733957924322Disabling writes for close at 1733957924323 (+1 ms)Writing region close event to WAL at 1733957924323Closed at 1733957924323 2024-12-11T22:58:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741883_1061 (size=93) 2024-12-11T22:58:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741883_1061 (size=93) 2024-12-11T22:58:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741883_1061 (size=93) 2024-12-11T22:58:44,329 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:44,329 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733957924206) 2024-12-11T22:58:44,343 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 408) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_822730104_22 at /127.0.0.1:41792 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:38049 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (80400340) connection to localhost/127.0.0.1:38049 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_822730104_22 at /127.0.0.1:38120 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 119) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11519 (was 11604) 2024-12-11T22:58:44,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=11518 2024-12-11T22:58:44,370 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:44,375 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T22:58:44,380 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b48762765ec4,38869,1733957909474 2024-12-11T22:58:44,383 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c42f9a1 2024-12-11T22:58:44,383 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T22:58:44,386 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T22:58:44,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T22:58:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-11T22:58:44,398 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T22:58:44,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-11T22:58:44,401 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:44,403 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T22:58:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:58:44,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741885_1063 (size=694) 2024-12-11T22:58:44,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741885_1063 (size=694) 2024-12-11T22:58:44,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741885_1063 (size=694) 2024-12-11T22:58:44,418 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b3825d650b99967f1eea9af6aad9717b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c 2024-12-11T22:58:44,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741886_1064 (size=77) 2024-12-11T22:58:44,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741886_1064 (size=77) 2024-12-11T22:58:44,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741886_1064 (size=77) 2024-12-11T22:58:44,427 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:44,427 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing b3825d650b99967f1eea9af6aad9717b, disabling compactions & flushes 2024-12-11T22:58:44,427 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:44,427 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:44,427 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. after waiting 0 ms 2024-12-11T22:58:44,427 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:44,427 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:44,427 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for b3825d650b99967f1eea9af6aad9717b: Waiting for close lock at 1733957924427Disabling compacts and flushes for region at 1733957924427Disabling writes for close at 1733957924427Writing region close event to WAL at 1733957924427Closed at 1733957924427 2024-12-11T22:58:44,429 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T22:58:44,436 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733957924429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733957924429"}]},"ts":"1733957924429"} 2024-12-11T22:58:44,439 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T22:58:44,440 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T22:58:44,443 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733957924440"}]},"ts":"1733957924440"} 2024-12-11T22:58:44,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-11T22:58:44,448 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {b48762765ec4=0} racks are {/default-rack=0} 2024-12-11T22:58:44,449 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:58:44,449 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:58:44,449 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-11T22:58:44,449 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:58:44,449 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:58:44,449 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-11T22:58:44,449 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:58:44,449 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:58:44,449 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-11T22:58:44,449 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:58:44,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN}] 2024-12-11T22:58:44,452 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN 2024-12-11T22:58:44,454 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN; state=OFFLINE, location=b48762765ec4,37687,1733957910157; forceNewPlan=false, retain=false 2024-12-11T22:58:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:58:44,606 INFO [b48762765ec4:38869 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-11T22:58:44,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPENING, regionLocation=b48762765ec4,37687,1733957910157 2024-12-11T22:58:44,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN because future has completed 2024-12-11T22:58:44,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157}] 2024-12-11T22:58:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:58:44,769 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:44,770 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b3825d650b99967f1eea9af6aad9717b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:44,770 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,770 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:44,770 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,770 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,772 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,773 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf1 2024-12-11T22:58:44,774 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:44,774 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:44,774 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,775 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf2 2024-12-11T22:58:44,776 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:44,776 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:44,776 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,777 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,777 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,778 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,778 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,779 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-11T22:58:44,780 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,782 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:44,783 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b3825d650b99967f1eea9af6aad9717b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58753801, jitterRate=-0.12450014054775238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-11T22:58:44,783 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:44,784 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b3825d650b99967f1eea9af6aad9717b: Running coprocessor pre-open hook at 1733957924770Writing region info on filesystem at 1733957924770Initializing all the Stores at 1733957924771 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957924771Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957924772 (+1 ms)Cleaning up temporary data from old regions at 1733957924778 (+6 ms)Running coprocessor post-open hooks at 1733957924783 (+5 ms)Region opened successfully at 1733957924784 (+1 ms) 2024-12-11T22:58:44,785 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., pid=6, masterSystemTime=1733957924764 2024-12-11T22:58:44,789 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:44,789 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:44,790 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPEN, openSeqNum=2, regionLocation=b48762765ec4,37687,1733957910157 2024-12-11T22:58:44,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157 because future has completed 2024-12-11T22:58:44,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T22:58:44,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157 in 183 msec 2024-12-11T22:58:44,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-11T22:58:44,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN in 348 msec 2024-12-11T22:58:44,803 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T22:58:44,803 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733957924803"}]},"ts":"1733957924803"} 2024-12-11T22:58:44,805 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-11T22:58:44,807 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T22:58:44,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 416 msec 2024-12-11T22:58:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T22:58:45,039 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-11T22:58:45,039 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-11T22:58:45,040 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:58:45,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-11T22:58:45,045 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T22:58:45,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-11T22:58:45,057 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=2] 2024-12-11T22:58:45,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=b3825d650b99967f1eea9af6aad9717b, source=b48762765ec4,37687,1733957910157, destination=b48762765ec4,34571,1733957910258, warming up region on b48762765ec4,34571,1733957910258 2024-12-11T22:58:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:58:45,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=b3825d650b99967f1eea9af6aad9717b, source=b48762765ec4,37687,1733957910157, destination=b48762765ec4,34571,1733957910258, running balancer 2024-12-11T22:58:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE 2024-12-11T22:58:45,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36883, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:58:45,077 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE 2024-12-11T22:58:45,080 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=CLOSING, regionLocation=b48762765ec4,37687,1733957910157 2024-12-11T22:58:45,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(7855): Warmup {ENCODED => b3825d650b99967f1eea9af6aad9717b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:45,083 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE because future has completed 2024-12-11T22:58:45,085 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T22:58:45,085 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf1 2024-12-11T22:58:45,085 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:45,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157}] 2024-12-11T22:58:45,085 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:45,086 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,087 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf2 2024-12-11T22:58:45,087 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:45,087 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(1722): Closing b3825d650b99967f1eea9af6aad9717b, disabling compactions & flushes 2024-12-11T22:58:45,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. after waiting 0 ms 2024-12-11T22:58:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] regionserver.HRegion(1676): Region close journal for b3825d650b99967f1eea9af6aad9717b: Waiting for close lock at 1733957925087Disabling compacts and flushes for region at 1733957925087Disabling writes for close at 1733957925088 (+1 ms)Writing region close event to WAL at 1733957925088Closed at 1733957925088 2024-12-11T22:58:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-11T22:58:45,244 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,245 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T22:58:45,245 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing b3825d650b99967f1eea9af6aad9717b, disabling compactions & flushes 2024-12-11T22:58:45,245 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,245 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,245 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. after waiting 0 ms 2024-12-11T22:58:45,245 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,246 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing b3825d650b99967f1eea9af6aad9717b 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-11T22:58:45,264 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf1/791938d72b4f4338af8a4abc0f5b3a05 is 35, key is r1/cf1:q/1733957925059/Put/seqid=0 2024-12-11T22:58:45,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741887_1065 (size=4783) 2024-12-11T22:58:45,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741887_1065 (size=4783) 2024-12-11T22:58:45,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741887_1065 (size=4783) 2024-12-11T22:58:45,272 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf1/791938d72b4f4338af8a4abc0f5b3a05 2024-12-11T22:58:45,280 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf1/791938d72b4f4338af8a4abc0f5b3a05 as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05 2024-12-11T22:58:45,286 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05, entries=1, sequenceid=5, filesize=4.7 K 2024-12-11T22:58:45,287 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for b3825d650b99967f1eea9af6aad9717b in 41ms, sequenceid=5, compaction requested=false 2024-12-11T22:58:45,287 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-11T22:58:45,292 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-11T22:58:45,295 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,295 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for b3825d650b99967f1eea9af6aad9717b: Waiting for close lock at 1733957925245Running coprocessor pre-close hooks at 1733957925245Disabling compacts and flushes for region at 1733957925245Disabling writes for close at 1733957925245Obtaining lock to block concurrent updates at 1733957925246 (+1 ms)Preparing flush snapshotting stores in b3825d650b99967f1eea9af6aad9717b at 1733957925246Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733957925247 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. at 1733957925247Flushing b3825d650b99967f1eea9af6aad9717b/cf1: creating writer at 1733957925247Flushing b3825d650b99967f1eea9af6aad9717b/cf1: appending metadata at 1733957925264 (+17 ms)Flushing b3825d650b99967f1eea9af6aad9717b/cf1: closing flushed file at 1733957925264Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61b3ebd4: reopening flushed file at 1733957925279 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for b3825d650b99967f1eea9af6aad9717b in 41ms, sequenceid=5, compaction requested=false at 1733957925287 (+8 ms)Writing region close event to WAL at 1733957925288 (+1 ms)Running coprocessor post-close hooks at 1733957925293 (+5 ms)Closed at 1733957925295 (+2 ms) 2024-12-11T22:58:45,295 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding b3825d650b99967f1eea9af6aad9717b move to b48762765ec4,34571,1733957910258 record at close sequenceid=5 2024-12-11T22:58:45,299 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,299 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=CLOSED 2024-12-11T22:58:45,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157 because future has completed 2024-12-11T22:58:45,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-11T22:58:45,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157 in 218 msec 2024-12-11T22:58:45,307 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE; state=CLOSED, location=b48762765ec4,34571,1733957910258; forceNewPlan=false, retain=false 2024-12-11T22:58:45,457 INFO [b48762765ec4:38869 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-11T22:58:45,458 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPENING, regionLocation=b48762765ec4,34571,1733957910258 2024-12-11T22:58:45,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE because future has completed 2024-12-11T22:58:45,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258}] 2024-12-11T22:58:45,619 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,619 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b3825d650b99967f1eea9af6aad9717b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:45,619 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,619 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:45,619 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,619 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,621 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,622 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf1 2024-12-11T22:58:45,622 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:45,629 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05 2024-12-11T22:58:45,629 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:45,629 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,630 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf2 2024-12-11T22:58:45,630 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:45,630 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:45,630 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,631 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,632 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,633 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,633 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,634 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-11T22:58:45,635 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,636 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b3825d650b99967f1eea9af6aad9717b; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66105032, jitterRate=-0.01495826244354248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-11T22:58:45,636 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:45,637 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b3825d650b99967f1eea9af6aad9717b: Running coprocessor pre-open hook at 1733957925620Writing region info on filesystem at 1733957925620Initializing all the Stores at 1733957925621 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957925621Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957925621Cleaning up temporary data from old regions at 1733957925633 (+12 ms)Running coprocessor post-open hooks at 1733957925636 (+3 ms)Region opened successfully at 1733957925637 (+1 ms) 2024-12-11T22:58:45,638 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., pid=9, masterSystemTime=1733957925614 2024-12-11T22:58:45,640 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,641 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,642 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPEN, openSeqNum=9, regionLocation=b48762765ec4,34571,1733957910258 2024-12-11T22:58:45,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258 because future has completed 2024-12-11T22:58:45,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-11T22:58:45,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258 in 185 msec 2024-12-11T22:58:45,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE in 573 msec 2024-12-11T22:58:45,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:58:45,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:58:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:41170 deadline: 1733957985690, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=34571 startCode=1733957910258. As of locationSeqNum=5. 2024-12-11T22:58:45,697 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=34571 startCode=1733957910258. As of locationSeqNum=5. 2024-12-11T22:58:45,698 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=34571 startCode=1733957910258. As of locationSeqNum=5. 2024-12-11T22:58:45,698 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,34571,1733957910258, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=34571 startCode=1733957910258. As of locationSeqNum=5. 2024-12-11T22:58:45,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:58:45,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:58:45,819 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b3825d650b99967f1eea9af6aad9717b 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-11T22:58:45,839 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf1/e2ae31c9d4094b73bd7544acf5c1360a is 29, key is r1/cf1:/1733957925810/DeleteFamily/seqid=0 2024-12-11T22:58:45,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741888_1066 (size=4906) 2024-12-11T22:58:45,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741888_1066 (size=4906) 2024-12-11T22:58:45,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741888_1066 (size=4906) 2024-12-11T22:58:45,848 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf1/e2ae31c9d4094b73bd7544acf5c1360a 2024-12-11T22:58:45,856 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2ae31c9d4094b73bd7544acf5c1360a 2024-12-11T22:58:45,872 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf2/dbd87b9907144c1dbb091e864e04f47c is 29, key is r1/cf2:/1733957925810/DeleteFamily/seqid=0 2024-12-11T22:58:45,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741889_1067 (size=4906) 2024-12-11T22:58:45,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741889_1067 (size=4906) 2024-12-11T22:58:45,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741889_1067 (size=4906) 2024-12-11T22:58:45,881 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf2/dbd87b9907144c1dbb091e864e04f47c 2024-12-11T22:58:45,888 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dbd87b9907144c1dbb091e864e04f47c 2024-12-11T22:58:45,889 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf1/e2ae31c9d4094b73bd7544acf5c1360a as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/e2ae31c9d4094b73bd7544acf5c1360a 2024-12-11T22:58:45,896 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2ae31c9d4094b73bd7544acf5c1360a 2024-12-11T22:58:45,896 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/e2ae31c9d4094b73bd7544acf5c1360a, entries=1, sequenceid=12, filesize=4.8 K 2024-12-11T22:58:45,898 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf2/dbd87b9907144c1dbb091e864e04f47c as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/dbd87b9907144c1dbb091e864e04f47c 2024-12-11T22:58:45,905 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dbd87b9907144c1dbb091e864e04f47c 2024-12-11T22:58:45,905 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/dbd87b9907144c1dbb091e864e04f47c, entries=1, sequenceid=12, filesize=4.8 K 2024-12-11T22:58:45,907 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for b3825d650b99967f1eea9af6aad9717b in 87ms, sequenceid=12, compaction requested=false 2024-12-11T22:58:45,907 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b3825d650b99967f1eea9af6aad9717b: 2024-12-11T22:58:45,910 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-11T22:58:45,911 DEBUG [Time-limited test {}] regionserver.HStore(1541): b3825d650b99967f1eea9af6aad9717b/cf1 is initiating major compaction (all files) 2024-12-11T22:58:45,912 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:58:45,912 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:45,912 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b3825d650b99967f1eea9af6aad9717b/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,913 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05, hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/e2ae31c9d4094b73bd7544acf5c1360a] into tmpdir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp, totalSize=9.5 K 2024-12-11T22:58:45,914 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 791938d72b4f4338af8a4abc0f5b3a05, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733957925059 2024-12-11T22:58:45,915 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e2ae31c9d4094b73bd7544acf5c1360a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-11T22:58:45,930 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): b3825d650b99967f1eea9af6aad9717b#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T22:58:45,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741890_1068 (size=4626) 2024-12-11T22:58:45,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741890_1068 (size=4626) 2024-12-11T22:58:45,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741890_1068 (size=4626) 2024-12-11T22:58:45,946 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf1/f9dd03013a6d4fb3b6796dc273a55756 as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/f9dd03013a6d4fb3b6796dc273a55756 2024-12-11T22:58:45,961 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in b3825d650b99967f1eea9af6aad9717b/cf1 of b3825d650b99967f1eea9af6aad9717b into f9dd03013a6d4fb3b6796dc273a55756(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T22:58:45,961 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b3825d650b99967f1eea9af6aad9717b: 2024-12-11T22:58:45,961 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-11T22:58:45,961 DEBUG [Time-limited test {}] regionserver.HStore(1541): b3825d650b99967f1eea9af6aad9717b/cf2 is initiating major compaction (all files) 2024-12-11T22:58:45,961 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T22:58:45,962 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T22:58:45,962 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b3825d650b99967f1eea9af6aad9717b/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:45,962 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/dbd87b9907144c1dbb091e864e04f47c] into tmpdir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp, totalSize=4.8 K 2024-12-11T22:58:45,963 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting dbd87b9907144c1dbb091e864e04f47c, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-11T22:58:45,969 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): b3825d650b99967f1eea9af6aad9717b#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T22:58:45,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741891_1069 (size=4592) 2024-12-11T22:58:45,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741891_1069 (size=4592) 2024-12-11T22:58:45,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741891_1069 (size=4592) 2024-12-11T22:58:45,986 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/.tmp/cf2/c1d036a0ff3643bf89dc40c75afa081d as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/c1d036a0ff3643bf89dc40c75afa081d 2024-12-11T22:58:45,993 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in b3825d650b99967f1eea9af6aad9717b/cf2 of b3825d650b99967f1eea9af6aad9717b into c1d036a0ff3643bf89dc40c75afa081d(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T22:58:45,994 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b3825d650b99967f1eea9af6aad9717b: 2024-12-11T22:58:45,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=b3825d650b99967f1eea9af6aad9717b, source=b48762765ec4,34571,1733957910258, destination=b48762765ec4,37687,1733957910157, warming up region on b48762765ec4,37687,1733957910157 2024-12-11T22:58:45,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=b3825d650b99967f1eea9af6aad9717b, source=b48762765ec4,34571,1733957910258, destination=b48762765ec4,37687,1733957910157, running balancer 2024-12-11T22:58:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE 2024-12-11T22:58:46,000 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE 2024-12-11T22:58:46,002 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=CLOSING, regionLocation=b48762765ec4,34571,1733957910258 2024-12-11T22:58:46,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(7855): Warmup {ENCODED => b3825d650b99967f1eea9af6aad9717b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:46,003 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,004 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf1 2024-12-11T22:58:46,004 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:46,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE because future has completed 2024-12-11T22:58:46,005 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T22:58:46,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258}] 2024-12-11T22:58:46,013 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05 2024-12-11T22:58:46,018 INFO [StoreFileOpener-b3825d650b99967f1eea9af6aad9717b-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2ae31c9d4094b73bd7544acf5c1360a 2024-12-11T22:58:46,018 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/e2ae31c9d4094b73bd7544acf5c1360a 2024-12-11T22:58:46,024 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/f9dd03013a6d4fb3b6796dc273a55756 2024-12-11T22:58:46,024 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:46,024 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,025 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf2 2024-12-11T22:58:46,025 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:46,033 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/c1d036a0ff3643bf89dc40c75afa081d 2024-12-11T22:58:46,038 INFO [StoreFileOpener-b3825d650b99967f1eea9af6aad9717b-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dbd87b9907144c1dbb091e864e04f47c 2024-12-11T22:58:46,038 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/dbd87b9907144c1dbb091e864e04f47c 2024-12-11T22:58:46,039 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(1722): Closing b3825d650b99967f1eea9af6aad9717b, disabling compactions & flushes 2024-12-11T22:58:46,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. after waiting 0 ms 2024-12-11T22:58:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37687 {}] regionserver.HRegion(1676): Region close journal for b3825d650b99967f1eea9af6aad9717b: Waiting for close lock at 1733957926039Disabling compacts and flushes for region at 1733957926039Disabling writes for close at 1733957926039Writing region close event to WAL at 1733957926040 (+1 ms)Closed at 1733957926040 2024-12-11T22:58:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-11T22:58:46,160 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,160 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T22:58:46,160 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing b3825d650b99967f1eea9af6aad9717b, disabling compactions & flushes 2024-12-11T22:58:46,161 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,161 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,161 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. after waiting 0 ms 2024-12-11T22:58:46,161 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,161 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05, hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/e2ae31c9d4094b73bd7544acf5c1360a] to archive 2024-12-11T22:58:46,165 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T22:58:46,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/e2ae31c9d4094b73bd7544acf5c1360a to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/e2ae31c9d4094b73bd7544acf5c1360a 2024-12-11T22:58:46,169 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05 to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/791938d72b4f4338af8a4abc0f5b3a05 2024-12-11T22:58:46,186 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/dbd87b9907144c1dbb091e864e04f47c] to archive 2024-12-11T22:58:46,188 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T22:58:46,190 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/dbd87b9907144c1dbb091e864e04f47c to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/dbd87b9907144c1dbb091e864e04f47c 2024-12-11T22:58:46,196 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-11T22:58:46,197 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,197 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for b3825d650b99967f1eea9af6aad9717b: Waiting for close lock at 1733957926160Running coprocessor pre-close hooks at 1733957926160Disabling compacts and flushes for region at 1733957926160Disabling writes for close at 1733957926161 (+1 ms)Writing region close event to WAL at 1733957926192 (+31 ms)Running coprocessor post-close hooks at 1733957926197 (+5 ms)Closed at 1733957926197 2024-12-11T22:58:46,197 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding b3825d650b99967f1eea9af6aad9717b move to b48762765ec4,37687,1733957910157 record at close sequenceid=12 2024-12-11T22:58:46,200 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,201 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=CLOSED 2024-12-11T22:58:46,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258 because future has completed 2024-12-11T22:58:46,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-11T22:58:46,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258 in 201 msec 2024-12-11T22:58:46,210 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE; state=CLOSED, location=b48762765ec4,37687,1733957910157; forceNewPlan=false, retain=false 2024-12-11T22:58:46,360 INFO [b48762765ec4:38869 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-11T22:58:46,361 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPENING, regionLocation=b48762765ec4,37687,1733957910157 2024-12-11T22:58:46,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE because future has completed 2024-12-11T22:58:46,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157}] 2024-12-11T22:58:46,521 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,521 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => b3825d650b99967f1eea9af6aad9717b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:46,521 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,521 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:46,522 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,522 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,523 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,524 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf1 2024-12-11T22:58:46,524 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:46,531 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/f9dd03013a6d4fb3b6796dc273a55756 2024-12-11T22:58:46,531 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:46,531 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,532 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf2 2024-12-11T22:58:46,532 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:46,539 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/c1d036a0ff3643bf89dc40c75afa081d 2024-12-11T22:58:46,539 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:46,539 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,540 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,542 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,543 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,543 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,544 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-11T22:58:46,546 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,547 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened b3825d650b99967f1eea9af6aad9717b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63102559, jitterRate=-0.059698596596717834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-11T22:58:46,547 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,548 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for b3825d650b99967f1eea9af6aad9717b: Running coprocessor pre-open hook at 1733957926522Writing region info on filesystem at 1733957926522Initializing all the Stores at 1733957926523 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957926523Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957926523Cleaning up temporary data from old regions at 1733957926543 (+20 ms)Running coprocessor post-open hooks at 1733957926547 (+4 ms)Region opened successfully at 1733957926548 (+1 ms) 2024-12-11T22:58:46,549 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., pid=12, masterSystemTime=1733957926517 2024-12-11T22:58:46,551 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,552 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,552 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPEN, openSeqNum=18, regionLocation=b48762765ec4,37687,1733957910157 2024-12-11T22:58:46,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157 because future has completed 2024-12-11T22:58:46,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-11T22:58:46,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,37687,1733957910157 in 192 msec 2024-12-11T22:58:46,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, REOPEN/MOVE in 561 msec 2024-12-11T22:58:46,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:58:46,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:58:46,606 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b48762765ec4,37687,1733957910157: testing ***** 2024-12-11T22:58:46,606 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-11T22:58:46,609 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-11T22:58:46,610 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-11T22:58:46,612 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-11T22:58:46,613 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-11T22:58:46,620 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 266906152 }, "NonHeapMemoryUsage": { "committed": 170655744, "init": 7667712, "max": -1, "used": 167967912 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b48762765ec4", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b48762765ec4", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52314", "tag.serverName": "b48762765ec4,33507,1733957910297", "tag.clusterId": "55e41239-6683-4c79-8b12-6cb06edb1ab3", "tag.Context": "regionserver", "tag.Hostname": "b48762765ec4", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733957910297, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-11T22:58:46,623 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38869 {}] master.MasterRpcServices(700): b48762765ec4,37687,1733957910157 reported a fatal error: ***** ABORTING region server b48762765ec4,37687,1733957910157: testing ***** 2024-12-11T22:58:46,626 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b48762765ec4,37687,1733957910157' ***** 2024-12-11T22:58:46,626 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-11T22:58:46,626 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T22:58:46,627 INFO [RS:0;b48762765ec4:37687 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-11T22:58:46,627 INFO [RS:0;b48762765ec4:37687 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-11T22:58:46,627 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T22:58:46,627 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(3091): Received CLOSE for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34571 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:45114 deadline: 1733957986627, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=37687 startCode=1733957910157. As of locationSeqNum=12. 2024-12-11T22:58:46,627 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(956): aborting server b48762765ec4,37687,1733957910157 2024-12-11T22:58:46,627 INFO [RS:0;b48762765ec4:37687 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:58:46,628 INFO [RS:0;b48762765ec4:37687 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b48762765ec4:37687. 2024-12-11T22:58:46,628 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b3825d650b99967f1eea9af6aad9717b, disabling compactions & flushes 2024-12-11T22:58:46,628 DEBUG [RS:0;b48762765ec4:37687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:58:46,628 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,628 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,34571,1733957910258, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,34571,1733957910258, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=37687 startCode=1733957910157. As of locationSeqNum=12. 2024-12-11T22:58:46,628 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,628 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. after waiting 0 ms 2024-12-11T22:58:46,628 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,34571,1733957910258, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=37687 startCode=1733957910157. As of locationSeqNum=12. 2024-12-11T22:58:46,628 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,628 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,34571,1733957910258, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b48762765ec4 port=37687 startCode=1733957910157. As of locationSeqNum=12. 2024-12-11T22:58:46,628 DEBUG [RS:0;b48762765ec4:37687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:58:46,629 INFO [RS:0;b48762765ec4:37687 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T22:58:46,629 INFO [RS:0;b48762765ec4:37687 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T22:58:46,629 INFO [RS:0;b48762765ec4:37687 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T22:58:46,629 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T22:58:46,629 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-11T22:58:46,629 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, b3825d650b99967f1eea9af6aad9717b=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.} 2024-12-11T22:58:46,629 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T22:58:46,629 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T22:58:46,629 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T22:58:46,630 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T22:58:46,630 DEBUG [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:46,630 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T22:58:46,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b48762765ec4,37687,1733957910157 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:46,630 ERROR [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-11T22:58:46,630 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,630 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b3825d650b99967f1eea9af6aad9717b: Waiting for close lock at 1733957926627Running coprocessor pre-close hooks at 1733957926628 (+1 ms)Disabling compacts and flushes for region at 1733957926628Disabling writes for close at 1733957926628Writing region close event to WAL at 1733957926630 (+2 ms)Running coprocessor post-close hooks at 1733957926630Closed at 1733957926630 2024-12-11T22:58:46,630 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T22:58:46,631 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:46,631 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T22:58:46,631 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733957926629Running coprocessor pre-close hooks at 1733957926629Disabling compacts and flushes for region at 1733957926629Disabling writes for close at 1733957926630 (+1 ms)Writing region close event to WAL at 1733957926630Running coprocessor post-close hooks at 1733957926630Closed at 1733957926631 (+1 ms) 2024-12-11T22:58:46,631 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T22:58:46,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b48762765ec4,37687,1733957910157 aborting 2024-12-11T22:58:46,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b48762765ec4,37687,1733957910157 aborting 2024-12-11T22:58:46,631 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 from cache 2024-12-11T22:58:46,708 INFO [regionserver/b48762765ec4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:58:46,737 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:58:46,739 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1] 2024-12-11T22:58:46,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b48762765ec4,37687,1733957910157 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:46,740 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b48762765ec4,37687,1733957910157 aborting 2024-12-11T22:58:46,740 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b48762765ec4,37687,1733957910157 aborting 2024-12-11T22:58:46,740 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 from cache 2024-12-11T22:58:46,830 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(976): stopping server b48762765ec4,37687,1733957910157; all regions closed. 2024-12-11T22:58:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741836_1012 (size=6845) 2024-12-11T22:58:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741836_1012 (size=6845) 2024-12-11T22:58:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741836_1012 (size=6845) 2024-12-11T22:58:46,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741835_1011 (size=2142) 2024-12-11T22:58:46,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741835_1011 (size=2142) 2024-12-11T22:58:46,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741835_1011 (size=2142) 2024-12-11T22:58:46,839 DEBUG [RS:0;b48762765ec4:37687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:58:46,839 INFO [RS:0;b48762765ec4:37687 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:58:46,839 INFO [RS:0;b48762765ec4:37687 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T22:58:46,839 INFO [RS:0;b48762765ec4:37687 {}] hbase.ChoreService(370): Chore service for: regionserver/b48762765ec4:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-11T22:58:46,839 INFO [RS:0;b48762765ec4:37687 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T22:58:46,839 INFO [regionserver/b48762765ec4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T22:58:46,840 INFO [RS:0;b48762765ec4:37687 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37687 2024-12-11T22:58:46,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T22:58:46,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b48762765ec4,37687,1733957910157 2024-12-11T22:58:46,848 INFO [RS:0;b48762765ec4:37687 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T22:58:46,851 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b48762765ec4,37687,1733957910157] 2024-12-11T22:58:46,852 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b48762765ec4,37687,1733957910157 already deleted, retry=false 2024-12-11T22:58:46,852 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b48762765ec4,37687,1733957910157 on b48762765ec4,38869,1733957909474 2024-12-11T22:58:46,858 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b48762765ec4,37687,1733957910157, splitWal=true, meta=true 2024-12-11T22:58:46,860 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for b48762765ec4,37687,1733957910157 (carryingMeta=true) b48762765ec4,37687,1733957910157/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7039ab00[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-11T22:58:46,861 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b48762765ec4,37687,1733957910157, splitWal=true, meta=true 2024-12-11T22:58:46,862 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure b48762765ec4,37687,1733957910157, splitWal=true, meta=true, isMeta: true 2024-12-11T22:58:46,864 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting 2024-12-11T22:58:46,865 INFO [PEWorker-1 {}] master.SplitWALManager(105): b48762765ec4,37687,1733957910157 WAL count=1, meta=true 2024-12-11T22:58:46,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta}] 2024-12-11T22:58:46,873 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=b48762765ec4,33507,1733957910297 2024-12-11T22:58:46,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, worker=b48762765ec4,33507,1733957910297}] 2024-12-11T22:58:46,947 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:58:46,949 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1] 2024-12-11T22:58:46,950 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b48762765ec4:37687 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b48762765ec4/172.17.0.3:37687 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:46,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:58:46,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37687-0x100cb900b300001, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:58:46,951 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1, error=java.net.ConnectException: Call to address=b48762765ec4:37687 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b48762765ec4/172.17.0.3:37687 2024-12-11T22:58:46,951 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-11T22:58:46,951 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 from cache 2024-12-11T22:58:46,951 INFO [RS:0;b48762765ec4:37687 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T22:58:46,951 INFO [RS:0;b48762765ec4:37687 {}] regionserver.HRegionServer(1031): Exiting; stopping=b48762765ec4,37687,1733957910157; zookeeper connection closed. 2024-12-11T22:58:46,952 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address b48762765ec4:37687 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b48762765ec4/172.17.0.3:37687 2024-12-11T22:58:46,952 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e0825f6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e0825f6 2024-12-11T22:58:47,035 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T22:58:47,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48443, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T22:58:47,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33507 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-11T22:58:47,058 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, size=6.7 K (6845bytes) 2024-12-11T22:58:47,058 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta 2024-12-11T22:58:47,059 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta after 1ms 2024-12-11T22:58:47,061 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta took 3ms 2024-12-11T22:58:47,069 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-11T22:58:47,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta so closing down 2024-12-11T22:58:47,070 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:47,071 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta.temp 2024-12-11T22:58:47,073 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta.temp 2024-12-11T22:58:47,073 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:47,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741892_1070 (size=6604) 2024-12-11T22:58:47,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741892_1070 (size=6604) 2024-12-11T22:58:47,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741892_1070 (size=6604) 2024-12-11T22:58:47,081 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:47,082 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta.temp to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-11T22:58:47,083 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-12-11T22:58:47,083 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, journal: Splitting hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, size=6.7 K (6845bytes) at 1733957927058Finishing writing output for hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta so closing down at 1733957927070 (+12 ms)Creating recovered edits writer path=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta.temp at 1733957927073 (+3 ms)3 split writer threads finished at 1733957927073Closed recovered edits writer path=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733957927081 (+8 ms)Rename recovered edits hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta.temp to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733957927082 (+1 ms)Processed 16 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1733957927083 (+1 ms) 2024-12-11T22:58:47,083 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta 2024-12-11T22:58:47,084 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-11T22:58:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-11T22:58:47,092 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs 2024-12-11T22:58:47,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-11T22:58:47,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, worker=b48762765ec4,33507,1733957910297 in 218 msec 2024-12-11T22:58:47,096 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=b48762765ec4,33507,1733957910297 2024-12-11T22:58:47,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-11T22:58:47,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b48762765ec4%2C37687%2C1733957910157.meta.1733957912114.meta, worker=b48762765ec4,33507,1733957910297 in 230 msec 2024-12-11T22:58:47,100 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting dir is empty, no logs to split. 2024-12-11T22:58:47,100 INFO [PEWorker-1 {}] master.SplitWALManager(105): b48762765ec4,37687,1733957910157 WAL count=0, meta=true 2024-12-11T22:58:47,100 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b48762765ec4,37687,1733957910157 WAL splitting is done? wals=0, meta=true 2024-12-11T22:58:47,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T22:58:47,103 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:58:47,104 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:58:47,254 DEBUG [b48762765ec4:38869 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-11T22:58:47,254 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(204): Hosts are {b48762765ec4=0} racks are {/default-rack=0} 2024-12-11T22:58:47,255 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:58:47,255 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:58:47,255 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:58:47,255 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:58:47,255 INFO [b48762765ec4:38869 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:58:47,255 INFO [b48762765ec4:38869 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:58:47,255 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:58:47,255 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b48762765ec4,33507,1733957910297 2024-12-11T22:58:47,256 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b48762765ec4,33507,1733957910297, state=OPENING 2024-12-11T22:58:47,257 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:58:47,258 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1] 2024-12-11T22:58:47,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:47,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:47,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:47,258 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T22:58:47,258 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:47,258 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:47,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b48762765ec4,33507,1733957910297}] 2024-12-11T22:58:47,259 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to b48762765ec4:37687 this server is in the failed servers list 2024-12-11T22:58:47,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:47,259 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b48762765ec4:37687 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b48762765ec4:37687 2024-12-11T22:58:47,259 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b48762765ec4:37687 2024-12-11T22:58:47,259 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 from cache 2024-12-11T22:58:47,415 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T22:58:47,415 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:47,416 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T22:58:47,417 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b48762765ec4%2C33507%2C1733957910297.meta, suffix=.meta, logDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,33507,1733957910297, archiveDir=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs, maxLogs=32 2024-12-11T22:58:47,431 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,33507,1733957910297/b48762765ec4%2C33507%2C1733957910297.meta.1733957927418.meta, exclude list is [], retry=0 2024-12-11T22:58:47,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:47,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:47,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:47,436 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,33507,1733957910297/b48762765ec4%2C33507%2C1733957910297.meta.1733957927418.meta 2024-12-11T22:58:47,437 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:47,437 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:47,438 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T22:58:47,438 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T22:58:47,438 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T22:58:47,438 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T22:58:47,438 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:47,438 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T22:58:47,438 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T22:58:47,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T22:58:47,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T22:58:47,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:47,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:47,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T22:58:47,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T22:58:47,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:47,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:47,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T22:58:47,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T22:58:47,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:47,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:47,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T22:58:47,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T22:58:47,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:47,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T22:58:47,447 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T22:58:47,447 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740 2024-12-11T22:58:47,448 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740 2024-12-11T22:58:47,449 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-11T22:58:47,454 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-11T22:58:47,454 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-11T22:58:47,475 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/info/f12f6dd5e64c4090bffe15e3e2d3b9ff is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b./info:regioninfo/1733957926552/Put/seqid=0 2024-12-11T22:58:47,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741894_1072 (size=11177) 2024-12-11T22:58:47,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741894_1072 (size=11177) 2024-12-11T22:58:47,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741894_1072 (size=11177) 2024-12-11T22:58:47,485 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/info/f12f6dd5e64c4090bffe15e3e2d3b9ff 2024-12-11T22:58:47,516 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/ns/7315696b83574bebb7bc3fcd18db3d6a is 43, key is default/ns:d/1733957912275/Put/seqid=0 2024-12-11T22:58:47,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741895_1073 (size=5153) 2024-12-11T22:58:47,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741895_1073 (size=5153) 2024-12-11T22:58:47,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741895_1073 (size=5153) 2024-12-11T22:58:47,525 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/ns/7315696b83574bebb7bc3fcd18db3d6a 2024-12-11T22:58:47,547 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/table/8baa46bb61c54085969abe710294a44e is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733957924803/Put/seqid=0 2024-12-11T22:58:47,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741896_1074 (size=5431) 2024-12-11T22:58:47,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741896_1074 (size=5431) 2024-12-11T22:58:47,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741896_1074 (size=5431) 2024-12-11T22:58:47,555 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/table/8baa46bb61c54085969abe710294a44e 2024-12-11T22:58:47,561 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/info/f12f6dd5e64c4090bffe15e3e2d3b9ff as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/info/f12f6dd5e64c4090bffe15e3e2d3b9ff 2024-12-11T22:58:47,567 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/info/f12f6dd5e64c4090bffe15e3e2d3b9ff, entries=36, sequenceid=18, filesize=10.9 K 2024-12-11T22:58:47,568 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/ns/7315696b83574bebb7bc3fcd18db3d6a as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/ns/7315696b83574bebb7bc3fcd18db3d6a 2024-12-11T22:58:47,575 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/ns/7315696b83574bebb7bc3fcd18db3d6a, entries=2, sequenceid=18, filesize=5.0 K 2024-12-11T22:58:47,576 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/table/8baa46bb61c54085969abe710294a44e as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/table/8baa46bb61c54085969abe710294a44e 2024-12-11T22:58:47,582 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/table/8baa46bb61c54085969abe710294a44e, entries=2, sequenceid=18, filesize=5.3 K 2024-12-11T22:58:47,582 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=18, compaction requested=false; wal=null 2024-12-11T22:58:47,583 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-11T22:58:47,584 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T22:58:47,584 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T22:58:47,585 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T22:58:47,587 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T22:58:47,589 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-11T22:58:47,590 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67037433, jitterRate=-0.001064404845237732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T22:58:47,590 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T22:58:47,591 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733957927439Writing region info on filesystem at 1733957927439Initializing all the Stores at 1733957927439Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957927440 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957927440Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957927440Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733957927440Obtaining lock to block concurrent updates at 1733957927454 (+14 ms)Preparing flush snapshotting stores in 1588230740 at 1733957927454Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733957927454Flushing stores of hbase:meta,,1.1588230740 at 1733957927454Flushing 1588230740/info: creating writer at 1733957927454Flushing 1588230740/info: appending metadata at 1733957927475 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733957927475Flushing 1588230740/ns: creating writer at 1733957927492 (+17 ms)Flushing 1588230740/ns: appending metadata at 1733957927515 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1733957927515Flushing 1588230740/table: creating writer at 1733957927531 (+16 ms)Flushing 1588230740/table: appending metadata at 1733957927546 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733957927547 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fcb5890: reopening flushed file at 1733957927561 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@599a29e9: reopening flushed file at 1733957927567 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5abfe077: reopening flushed file at 1733957927575 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=18, compaction requested=false; wal=null at 1733957927582 (+7 ms)Cleaning up temporary data from old regions at 1733957927584 (+2 ms)Running coprocessor post-open hooks at 1733957927590 (+6 ms)Region opened successfully at 1733957927591 (+1 ms) 2024-12-11T22:58:47,593 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733957927411 2024-12-11T22:58:47,595 DEBUG [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T22:58:47,595 INFO [RS_OPEN_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T22:58:47,596 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=b48762765ec4,33507,1733957910297 2024-12-11T22:58:47,597 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b48762765ec4,33507,1733957910297, state=OPEN 2024-12-11T22:58:47,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:47,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:47,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T22:58:47,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:47,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:47,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T22:58:47,599 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b48762765ec4,33507,1733957910297 2024-12-11T22:58:47,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-11T22:58:47,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b48762765ec4,33507,1733957910297 in 341 msec 2024-12-11T22:58:47,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-11T22:58:47,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 500 msec 2024-12-11T22:58:47,606 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): b48762765ec4,37687,1733957910157 had 2 regions 2024-12-11T22:58:47,607 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b48762765ec4,37687,1733957910157, splitWal=true, meta=true, isMeta: false 2024-12-11T22:58:47,609 INFO [PEWorker-2 {}] master.SplitWALManager(105): b48762765ec4,37687,1733957910157 WAL count=1, meta=false 2024-12-11T22:58:47,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b48762765ec4%2C37687%2C1733957910157.1733957911750}] 2024-12-11T22:58:47,611 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=b48762765ec4,33507,1733957910297 2024-12-11T22:58:47,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b48762765ec4%2C37687%2C1733957910157.1733957911750, worker=b48762765ec4,33507,1733957910297}] 2024-12-11T22:58:47,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33507 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-11T22:58:47,767 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:58:47,768 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b48762765ec4,33507,1733957910297, seqNum=-1] 2024-12-11T22:58:47,768 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:58:47,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:58:47,786 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750, size=2.1 K (2142bytes) 2024-12-11T22:58:47,786 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750 2024-12-11T22:58:47,787 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750 after 1ms 2024-12-11T22:58:47,789 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750 took 3ms 2024-12-11T22:58:47,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for b3825d650b99967f1eea9af6aad9717b: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-11T22:58:47,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750 so closing down 2024-12-11T22:58:47,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:47,792 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:47,793 INFO [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-12-11T22:58:47,793 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750, journal: Splitting hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750, size=2.1 K (2142bytes) at 1733957927786Finishing writing output for hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750 so closing down at 1733957927792 (+6 ms)3 split writer threads finished at 1733957927792Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1733957927793 (+1 ms) 2024-12-11T22:58:47,793 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750 2024-12-11T22:58:47,793 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b48762765ec4:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-11T22:58:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38869 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-11T22:58:47,797 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting/b48762765ec4%2C37687%2C1733957910157.1733957911750 to hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs 2024-12-11T22:58:47,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-11T22:58:47,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b48762765ec4%2C37687%2C1733957910157.1733957911750, worker=b48762765ec4,33507,1733957910297 in 185 msec 2024-12-11T22:58:47,801 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b48762765ec4,33507,1733957910297 2024-12-11T22:58:47,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-11T22:58:47,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b48762765ec4%2C37687%2C1733957910157.1733957911750, worker=b48762765ec4,33507,1733957910297 in 192 msec 2024-12-11T22:58:47,804 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/WALs/b48762765ec4,37687,1733957910157-splitting dir is empty, no logs to split. 2024-12-11T22:58:47,804 INFO [PEWorker-2 {}] master.SplitWALManager(105): b48762765ec4,37687,1733957910157 WAL count=0, meta=false 2024-12-11T22:58:47,804 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if b48762765ec4,37687,1733957910157 WAL splitting is done? wals=0, meta=false 2024-12-11T22:58:47,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN}] 2024-12-11T22:58:47,808 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN 2024-12-11T22:58:47,809 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-11T22:58:47,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to b48762765ec4:37687 this server is in the failed servers list 2024-12-11T22:58:47,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b48762765ec4:37687 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b48762765ec4:37687 2024-12-11T22:58:47,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b48762765ec4:37687 2024-12-11T22:58:47,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=12 from cache 2024-12-11T22:58:47,959 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(204): Hosts are {b48762765ec4=0} racks are {/default-rack=0} 2024-12-11T22:58:47,960 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-11T22:58:47,960 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-11T22:58:47,960 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-11T22:58:47,960 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-11T22:58:47,960 INFO [b48762765ec4:38869 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-11T22:58:47,960 INFO [b48762765ec4:38869 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-11T22:58:47,960 DEBUG [b48762765ec4:38869 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-11T22:58:47,960 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPENING, regionLocation=b48762765ec4,34571,1733957910258 2024-12-11T22:58:47,961 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b48762765ec4:37687 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b48762765ec4/172.17.0.3:37687 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:47,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1, error=java.net.ConnectException: Call to address=b48762765ec4:37687 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b48762765ec4/172.17.0.3:37687 2024-12-11T22:58:47,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-11T22:58:47,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b48762765ec4,37687,1733957910157, seqNum=-1 from cache 2024-12-11T22:58:47,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address b48762765ec4:37687 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b48762765ec4/172.17.0.3:37687 2024-12-11T22:58:48,067 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T22:58:48,067 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b48762765ec4,33507,1733957910297, seqNum=-1] 2024-12-11T22:58:48,068 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T22:58:48,069 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45431, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T22:58:48,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN because future has completed 2024-12-11T22:58:48,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258}] 2024-12-11T22:58:48,104 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=18] 2024-12-11T22:58:48,104 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to b48762765ec4:37687 this server is in the failed servers list 2024-12-11T22:58:48,105 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b48762765ec4:37687 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b48762765ec4:37687 2024-12-11T22:58:48,105 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b48762765ec4:37687 2024-12-11T22:58:48,105 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,37687,1733957910157, seqNum=18 from cache 2024-12-11T22:58:48,228 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:48,228 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => b3825d650b99967f1eea9af6aad9717b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:48,228 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,229 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:48,229 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,229 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,230 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,231 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf1 2024-12-11T22:58:48,231 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,238 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf1/f9dd03013a6d4fb3b6796dc273a55756 2024-12-11T22:58:48,238 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,238 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,239 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3825d650b99967f1eea9af6aad9717b columnFamilyName cf2 2024-12-11T22:58:48,239 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,246 DEBUG [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/cf2/c1d036a0ff3643bf89dc40c75afa081d 2024-12-11T22:58:48,246 INFO [StoreOpener-b3825d650b99967f1eea9af6aad9717b-1 {}] regionserver.HStore(327): Store=b3825d650b99967f1eea9af6aad9717b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,246 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,247 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,248 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,249 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,249 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,250 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-11T22:58:48,251 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,252 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened b3825d650b99967f1eea9af6aad9717b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72527112, jitterRate=0.08073818683624268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-11T22:58:48,252 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:58:48,253 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for b3825d650b99967f1eea9af6aad9717b: Running coprocessor pre-open hook at 1733957928229Writing region info on filesystem at 1733957928229Initializing all the Stores at 1733957928230 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928230Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928230Cleaning up temporary data from old regions at 1733957928249 (+19 ms)Running coprocessor post-open hooks at 1733957928252 (+3 ms)Region opened successfully at 1733957928253 (+1 ms) 2024-12-11T22:58:48,254 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., pid=21, masterSystemTime=1733957928224 2024-12-11T22:58:48,257 DEBUG [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:48,257 INFO [RS_OPEN_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:58:48,257 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b3825d650b99967f1eea9af6aad9717b, regionState=OPEN, openSeqNum=18, regionLocation=b48762765ec4,34571,1733957910258 2024-12-11T22:58:48,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258 because future has completed 2024-12-11T22:58:48,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-11T22:58:48,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure b3825d650b99967f1eea9af6aad9717b, server=b48762765ec4,34571,1733957910258 in 190 msec 2024-12-11T22:58:48,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-11T22:58:48,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=b3825d650b99967f1eea9af6aad9717b, ASSIGN in 458 msec 2024-12-11T22:58:48,268 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server b48762765ec4,37687,1733957910157 after splitting done 2024-12-11T22:58:48,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure b48762765ec4,37687,1733957910157, splitWal=true, meta=true in 1.4150 sec 2024-12-11T22:58:48,422 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b., hostname=b48762765ec4,34571,1733957910258, seqNum=18] 2024-12-11T22:58:48,440 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413 (was 413), OpenFileDescriptor=1050 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=11448 (was 11518) 2024-12-11T22:58:48,442 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-12-11T22:58:48,456 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=413, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=11447 2024-12-11T22:58:48,456 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-12-11T22:58:48,476 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:48,478 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:48,479 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:48,481 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33847477, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-33847477, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:48,495 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33847477/hregion-33847477.1733957928482, exclude list is [], retry=0 2024-12-11T22:58:48,498 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:48,498 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:48,498 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:48,502 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33847477/hregion-33847477.1733957928482 2024-12-11T22:58:48,502 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:48,502 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a75708d1ca19824d54c8a904aa37c908, NAME => 'testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:48,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741898_1076 (size=67) 2024-12-11T22:58:48,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741898_1076 (size=67) 2024-12-11T22:58:48,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741898_1076 (size=67) 2024-12-11T22:58:48,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:48,513 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,514 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName a 2024-12-11T22:58:48,514 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,515 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,515 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,516 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName b 2024-12-11T22:58:48,516 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,517 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,517 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,518 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName c 2024-12-11T22:58:48,518 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,518 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,519 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,521 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,521 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,521 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:48,522 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,524 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:48,524 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a75708d1ca19824d54c8a904aa37c908; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73429109, jitterRate=0.09417898952960968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:48,525 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a75708d1ca19824d54c8a904aa37c908: Writing region info on filesystem at 1733957928511Initializing all the Stores at 1733957928512 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928512Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928512Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928512Cleaning up temporary data from old regions at 1733957928521 (+9 ms)Region opened successfully at 1733957928525 (+4 ms) 2024-12-11T22:58:48,525 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a75708d1ca19824d54c8a904aa37c908, disabling compactions & flushes 2024-12-11T22:58:48,525 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,525 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,525 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. after waiting 0 ms 2024-12-11T22:58:48,525 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,526 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,526 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a75708d1ca19824d54c8a904aa37c908: Waiting for close lock at 1733957928525Disabling compacts and flushes for region at 1733957928525Disabling writes for close at 1733957928525Writing region close event to WAL at 1733957928526 (+1 ms)Closed at 1733957928526 2024-12-11T22:58:48,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741897_1075 (size=93) 2024-12-11T22:58:48,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741897_1075 (size=93) 2024-12-11T22:58:48,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741897_1075 (size=93) 2024-12-11T22:58:48,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:48,532 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33847477:(num 1733957928482) 2024-12-11T22:58:48,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:48,534 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:48,550 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534, exclude list is [], retry=0 2024-12-11T22:58:48,552 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:48,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:48,553 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:48,555 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534 2024-12-11T22:58:48,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:48,555 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a75708d1ca19824d54c8a904aa37c908, NAME => 'testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:48,555 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:48,555 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,555 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,557 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,558 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName a 2024-12-11T22:58:48,558 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,558 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,558 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,559 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName b 2024-12-11T22:58:48,559 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,560 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,560 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,560 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName c 2024-12-11T22:58:48,560 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,561 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,563 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,563 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,564 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:48,565 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,566 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a75708d1ca19824d54c8a904aa37c908; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59762217, jitterRate=-0.1094735711812973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:48,566 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a75708d1ca19824d54c8a904aa37c908: Writing region info on filesystem at 1733957928556Initializing all the Stores at 1733957928556Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928557 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928557Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928557Cleaning up temporary data from old regions at 1733957928563 (+6 ms)Region opened successfully at 1733957928566 (+3 ms) 2024-12-11T22:58:48,603 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a75708d1ca19824d54c8a904aa37c908 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-11T22:58:48,620 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/a/0f95d149d5d54cc6b7ee3f6b5e67142f is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733957928566/Put/seqid=0 2024-12-11T22:58:48,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741900_1078 (size=5958) 2024-12-11T22:58:48,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741900_1078 (size=5958) 2024-12-11T22:58:48,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741900_1078 (size=5958) 2024-12-11T22:58:48,633 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/a/0f95d149d5d54cc6b7ee3f6b5e67142f 2024-12-11T22:58:48,660 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/b/f5c805f746e94b6bb1a5e1d15a4eafeb is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733957928582/Put/seqid=0 2024-12-11T22:58:48,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741901_1079 (size=5958) 2024-12-11T22:58:48,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741901_1079 (size=5958) 2024-12-11T22:58:48,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741901_1079 (size=5958) 2024-12-11T22:58:48,667 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/b/f5c805f746e94b6bb1a5e1d15a4eafeb 2024-12-11T22:58:48,688 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/c/2af1b99d65e849fda8be59786b8cfc6e is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733957928593/Put/seqid=0 2024-12-11T22:58:48,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741902_1080 (size=5958) 2024-12-11T22:58:48,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741902_1080 (size=5958) 2024-12-11T22:58:48,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741902_1080 (size=5958) 2024-12-11T22:58:48,695 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/c/2af1b99d65e849fda8be59786b8cfc6e 2024-12-11T22:58:48,701 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/a/0f95d149d5d54cc6b7ee3f6b5e67142f as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/a/0f95d149d5d54cc6b7ee3f6b5e67142f 2024-12-11T22:58:48,705 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/a/0f95d149d5d54cc6b7ee3f6b5e67142f, entries=10, sequenceid=33, filesize=5.8 K 2024-12-11T22:58:48,706 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/b/f5c805f746e94b6bb1a5e1d15a4eafeb as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/b/f5c805f746e94b6bb1a5e1d15a4eafeb 2024-12-11T22:58:48,712 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/b/f5c805f746e94b6bb1a5e1d15a4eafeb, entries=10, sequenceid=33, filesize=5.8 K 2024-12-11T22:58:48,713 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/c/2af1b99d65e849fda8be59786b8cfc6e as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/c/2af1b99d65e849fda8be59786b8cfc6e 2024-12-11T22:58:48,718 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/c/2af1b99d65e849fda8be59786b8cfc6e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-11T22:58:48,719 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for a75708d1ca19824d54c8a904aa37c908 in 117ms, sequenceid=33, compaction requested=false 2024-12-11T22:58:48,719 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a75708d1ca19824d54c8a904aa37c908: 2024-12-11T22:58:48,719 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a75708d1ca19824d54c8a904aa37c908, disabling compactions & flushes 2024-12-11T22:58:48,719 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,719 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,719 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. after waiting 0 ms 2024-12-11T22:58:48,719 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,720 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. 2024-12-11T22:58:48,720 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a75708d1ca19824d54c8a904aa37c908: Waiting for close lock at 1733957928719Disabling compacts and flushes for region at 1733957928719Disabling writes for close at 1733957928719Writing region close event to WAL at 1733957928720 (+1 ms)Closed at 1733957928720 2024-12-11T22:58:48,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741899_1077 (size=5937) 2024-12-11T22:58:48,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741899_1077 (size=5937) 2024-12-11T22:58:48,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741899_1077 (size=5937) 2024-12-11T22:58:48,729 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/b/f5c805f746e94b6bb1a5e1d15a4eafeb to hdfs://localhost:38049/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/b/f5c805f746e94b6bb1a5e1d15a4eafeb 2024-12-11T22:58:48,745 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534, size=5.8 K (5937bytes) 2024-12-11T22:58:48,746 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534 2024-12-11T22:58:48,746 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534 after 0ms 2024-12-11T22:58:48,748 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534 took 3ms 2024-12-11T22:58:48,750 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534 so closing down 2024-12-11T22:58:48,750 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:48,751 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733957928534.temp 2024-12-11T22:58:48,753 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000003-wal.1733957928534.temp 2024-12-11T22:58:48,753 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741903_1081 (size=5223) 2024-12-11T22:58:48,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741903_1081 (size=5223) 2024-12-11T22:58:48,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741903_1081 (size=5223) 2024-12-11T22:58:48,761 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000003-wal.1733957928534.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:48,763 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000003-wal.1733957928534.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000032 2024-12-11T22:58:48,763 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-12-11T22:58:48,763 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534, journal: Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534, size=5.8 K (5937bytes) at 1733957928746Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534 so closing down at 1733957928750 (+4 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000003-wal.1733957928534.temp at 1733957928753 (+3 ms)3 split writer threads finished at 1733957928753Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000003-wal.1733957928534.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733957928761 (+8 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000003-wal.1733957928534.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000032 at 1733957928763 (+2 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1733957928763 2024-12-11T22:58:48,765 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928534 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957928534 2024-12-11T22:58:48,766 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000032 2024-12-11T22:58:48,766 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:48,767 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:48,782 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928768, exclude list is [], retry=0 2024-12-11T22:58:48,784 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:48,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:48,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:48,787 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928768 2024-12-11T22:58:48,787 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:48,787 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a75708d1ca19824d54c8a904aa37c908, NAME => 'testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:48,787 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:48,787 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,787 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,789 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,790 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName a 2024-12-11T22:58:48,790 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,796 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/a/0f95d149d5d54cc6b7ee3f6b5e67142f 2024-12-11T22:58:48,796 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,796 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,798 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName b 2024-12-11T22:58:48,798 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,799 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,799 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,800 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a75708d1ca19824d54c8a904aa37c908 columnFamilyName c 2024-12-11T22:58:48,800 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:48,807 DEBUG [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/c/2af1b99d65e849fda8be59786b8cfc6e 2024-12-11T22:58:48,807 INFO [StoreOpener-a75708d1ca19824d54c8a904aa37c908-1 {}] regionserver.HStore(327): Store=a75708d1ca19824d54c8a904aa37c908/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:48,808 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,808 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,810 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,811 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000032 2024-12-11T22:58:48,814 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000032 2024-12-11T22:58:48,814 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a75708d1ca19824d54c8a904aa37c908 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-11T22:58:48,838 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/b/b1249befe1374b6ab4f38e107a9fc0aa is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733957928582/Put/seqid=0 2024-12-11T22:58:48,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741905_1083 (size=5958) 2024-12-11T22:58:48,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741905_1083 (size=5958) 2024-12-11T22:58:48,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741905_1083 (size=5958) 2024-12-11T22:58:48,847 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/b/b1249befe1374b6ab4f38e107a9fc0aa 2024-12-11T22:58:48,856 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/.tmp/b/b1249befe1374b6ab4f38e107a9fc0aa as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/b/b1249befe1374b6ab4f38e107a9fc0aa 2024-12-11T22:58:48,862 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/b/b1249befe1374b6ab4f38e107a9fc0aa, entries=10, sequenceid=32, filesize=5.8 K 2024-12-11T22:58:48,862 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a75708d1ca19824d54c8a904aa37c908 in 48ms, sequenceid=32, compaction requested=false; wal=null 2024-12-11T22:58:48,863 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/0000000000000000032 2024-12-11T22:58:48,864 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,864 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,865 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:48,866 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a75708d1ca19824d54c8a904aa37c908 2024-12-11T22:58:48,868 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/a75708d1ca19824d54c8a904aa37c908/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-11T22:58:48,869 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a75708d1ca19824d54c8a904aa37c908; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58891344, jitterRate=-0.12245059013366699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:48,870 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a75708d1ca19824d54c8a904aa37c908: Writing region info on filesystem at 1733957928787Initializing all the Stores at 1733957928788 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928788Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928788Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957928788Obtaining lock to block concurrent updates at 1733957928815 (+27 ms)Preparing flush snapshotting stores in a75708d1ca19824d54c8a904aa37c908 at 1733957928815Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733957928815Flushing stores of testReplayEditsWrittenViaHRegion,,1733957928477.a75708d1ca19824d54c8a904aa37c908. at 1733957928815Flushing a75708d1ca19824d54c8a904aa37c908/b: creating writer at 1733957928815Flushing a75708d1ca19824d54c8a904aa37c908/b: appending metadata at 1733957928837 (+22 ms)Flushing a75708d1ca19824d54c8a904aa37c908/b: closing flushed file at 1733957928837Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f19eb7e: reopening flushed file at 1733957928852 (+15 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a75708d1ca19824d54c8a904aa37c908 in 48ms, sequenceid=32, compaction requested=false; wal=null at 1733957928862 (+10 ms)Cleaning up temporary data from old regions at 1733957928864 (+2 ms)Region opened successfully at 1733957928870 (+6 ms) 2024-12-11T22:58:48,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=421 (was 413) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:41792 [Waiting for operation #56] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:49278 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:45596 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:37274 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:37408 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:38120 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=125 (was 125), ProcessCount=11 (was 11), AvailableMemoryMB=11441 (was 11447) 2024-12-11T22:58:48,895 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-11T22:58:48,911 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=421, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=125, ProcessCount=11, AvailableMemoryMB=11440 2024-12-11T22:58:48,911 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-11T22:58:48,934 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:48,937 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:48,938 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:48,941 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36909496, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-36909496, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:48,955 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36909496/hregion-36909496.1733957928941, exclude list is [], retry=0 2024-12-11T22:58:48,958 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:48,958 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:48,958 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:48,960 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36909496/hregion-36909496.1733957928941 2024-12-11T22:58:48,962 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:48,962 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 228e6f3399ee99d920bd97c6b5418b49, NAME => 'testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:48,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741907_1085 (size=68) 2024-12-11T22:58:48,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741907_1085 (size=68) 2024-12-11T22:58:48,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741907_1085 (size=68) 2024-12-11T22:58:49,375 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:49,377 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,378 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName a 2024-12-11T22:58:49,378 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:49,379 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:49,379 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,380 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName b 2024-12-11T22:58:49,380 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:49,381 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:49,381 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,382 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName c 2024-12-11T22:58:49,382 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:49,382 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:49,382 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,383 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,384 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,384 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,385 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:49,385 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,387 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:49,388 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 228e6f3399ee99d920bd97c6b5418b49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67362066, jitterRate=0.0037730038166046143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:49,388 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 228e6f3399ee99d920bd97c6b5418b49: Writing region info on filesystem at 1733957929375Initializing all the Stores at 1733957929376 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957929376Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957929376Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957929376Cleaning up temporary data from old regions at 1733957929384 (+8 ms)Region opened successfully at 1733957929388 (+4 ms) 2024-12-11T22:58:49,388 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 228e6f3399ee99d920bd97c6b5418b49, disabling compactions & flushes 2024-12-11T22:58:49,388 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:49,388 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:49,388 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. after waiting 0 ms 2024-12-11T22:58:49,388 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:49,389 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:49,389 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 228e6f3399ee99d920bd97c6b5418b49: Waiting for close lock at 1733957929388Disabling compacts and flushes for region at 1733957929388Disabling writes for close at 1733957929388Writing region close event to WAL at 1733957929389 (+1 ms)Closed at 1733957929389 2024-12-11T22:58:49,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741906_1084 (size=93) 2024-12-11T22:58:49,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741906_1084 (size=93) 2024-12-11T22:58:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741906_1084 (size=93) 2024-12-11T22:58:49,394 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:49,394 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36909496:(num 1733957928941) 2024-12-11T22:58:49,394 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:49,396 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:49,409 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396, exclude list is [], retry=0 2024-12-11T22:58:49,411 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:49,412 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:49,412 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:49,413 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396 2024-12-11T22:58:49,414 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:49,481 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 228e6f3399ee99d920bd97c6b5418b49, NAME => 'testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:49,483 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,483 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:49,483 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,483 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,485 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,486 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName a 2024-12-11T22:58:49,487 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:49,487 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:49,487 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,488 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName b 2024-12-11T22:58:49,488 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:49,489 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:49,489 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,489 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName c 2024-12-11T22:58:49,490 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:49,490 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:49,490 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,493 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,493 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,494 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:49,495 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,496 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 228e6f3399ee99d920bd97c6b5418b49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59319722, jitterRate=-0.11606726050376892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:49,496 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:49,497 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 228e6f3399ee99d920bd97c6b5418b49: Running coprocessor pre-open hook at 1733957929483Writing region info on filesystem at 1733957929484 (+1 ms)Initializing all the Stores at 1733957929485 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957929485Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957929485Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957929485Cleaning up temporary data from old regions at 1733957929493 (+8 ms)Running coprocessor post-open hooks at 1733957929496 (+3 ms)Region opened successfully at 1733957929497 (+1 ms) 2024-12-11T22:58:49,511 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 228e6f3399ee99d920bd97c6b5418b49 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-11T22:58:49,512 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:50,299 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T22:58:50,330 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-11T22:58:50,330 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:50,330 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-11T22:58:50,330 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:50,331 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-11T22:58:50,331 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:50,332 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-11T22:58:50,332 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:50,513 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:51,514 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:52,515 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:53,515 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:54,516 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:55,516 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:56,517 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:57,518 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:58,518 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 228e6f3399ee99d920bd97c6b5418b49/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:58:58,520 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 228e6f3399ee99d920bd97c6b5418b49: 2024-12-11T22:58:58,520 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:58,537 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 228e6f3399ee99d920bd97c6b5418b49: 2024-12-11T22:58:58,537 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-11T22:58:58,537 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 228e6f3399ee99d920bd97c6b5418b49, disabling compactions & flushes 2024-12-11T22:58:58,537 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:58,537 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:58,537 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. after waiting 0 ms 2024-12-11T22:58:58,537 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:58,538 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:58,538 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. 2024-12-11T22:58:58,538 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 228e6f3399ee99d920bd97c6b5418b49: Waiting for close lock at 1733957938537Running coprocessor pre-close hooks at 1733957938537Disabling compacts and flushes for region at 1733957938537Disabling writes for close at 1733957938537Writing region close event to WAL at 1733957938538 (+1 ms)Running coprocessor post-close hooks at 1733957938538Closed at 1733957938538 2024-12-11T22:58:58,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741908_1086 (size=3941) 2024-12-11T22:58:58,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741908_1086 (size=3941) 2024-12-11T22:58:58,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741908_1086 (size=3941) 2024-12-11T22:58:58,567 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396, size=3.8 K (3941bytes) 2024-12-11T22:58:58,567 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396 2024-12-11T22:58:58,568 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396 after 1ms 2024-12-11T22:58:58,579 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396 took 12ms 2024-12-11T22:58:58,584 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396 so closing down 2024-12-11T22:58:58,584 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:58,585 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733957929396.temp 2024-12-11T22:58:58,592 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000004-wal.1733957929396.temp 2024-12-11T22:58:58,593 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:58,598 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T22:58:58,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741909_1087 (size=2983) 2024-12-11T22:58:58,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741909_1087 (size=2983) 2024-12-11T22:58:58,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741909_1087 (size=2983) 2024-12-11T22:58:58,611 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000004-wal.1733957929396.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:58,612 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000004-wal.1733957929396.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000026 2024-12-11T22:58:58,613 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-12-11T22:58:58,613 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396, journal: Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396, size=3.8 K (3941bytes) at 1733957938567Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396 so closing down at 1733957938584 (+17 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000004-wal.1733957929396.temp at 1733957938593 (+9 ms)3 split writer threads finished at 1733957938593Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000004-wal.1733957929396.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733957938611 (+18 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000004-wal.1733957929396.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000026 at 1733957938612 (+1 ms)Processed 23 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1733957938613 (+1 ms) 2024-12-11T22:58:58,615 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957929396 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957929396 2024-12-11T22:58:58,616 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000026 2024-12-11T22:58:58,616 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:58,619 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:58,634 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957938619, exclude list is [], retry=0 2024-12-11T22:58:58,638 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:58,638 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:58,638 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:58,641 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957938619 2024-12-11T22:58:58,649 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:58,649 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 228e6f3399ee99d920bd97c6b5418b49, NAME => 'testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:58,650 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,650 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:58,651 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,651 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,653 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,654 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName a 2024-12-11T22:58:58,655 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:58,656 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:58,656 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,657 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName b 2024-12-11T22:58:58,659 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:58,659 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:58,660 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,661 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 228e6f3399ee99d920bd97c6b5418b49 columnFamilyName c 2024-12-11T22:58:58,661 DEBUG [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:58,661 INFO [StoreOpener-228e6f3399ee99d920bd97c6b5418b49-1 {}] regionserver.HStore(327): Store=228e6f3399ee99d920bd97c6b5418b49/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:58,661 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,662 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,664 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,665 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000026 2024-12-11T22:58:58,675 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000026 2024-12-11T22:58:58,675 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 228e6f3399ee99d920bd97c6b5418b49 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-11T22:58:58,694 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/a/b9261b2544e84a2d88eced98227368d8 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733957938524/Put/seqid=0 2024-12-11T22:58:58,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741911_1089 (size=5523) 2024-12-11T22:58:58,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741911_1089 (size=5523) 2024-12-11T22:58:58,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741911_1089 (size=5523) 2024-12-11T22:58:58,727 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/a/b9261b2544e84a2d88eced98227368d8 2024-12-11T22:58:58,751 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/b/dfb70d6837b04d21b8209d75eb194ad1 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733957938521/Put/seqid=0 2024-12-11T22:58:58,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741912_1090 (size=5524) 2024-12-11T22:58:58,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741912_1090 (size=5524) 2024-12-11T22:58:58,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741912_1090 (size=5524) 2024-12-11T22:58:58,764 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/b/dfb70d6837b04d21b8209d75eb194ad1 2024-12-11T22:58:58,786 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/c/6f5c67a263044601b1f19ec9216c7c9f is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733957938522/Put/seqid=0 2024-12-11T22:58:58,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741913_1091 (size=5457) 2024-12-11T22:58:58,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741913_1091 (size=5457) 2024-12-11T22:58:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741913_1091 (size=5457) 2024-12-11T22:58:58,795 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/c/6f5c67a263044601b1f19ec9216c7c9f 2024-12-11T22:58:58,801 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/a/b9261b2544e84a2d88eced98227368d8 as hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/a/b9261b2544e84a2d88eced98227368d8 2024-12-11T22:58:58,807 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/a/b9261b2544e84a2d88eced98227368d8, entries=7, sequenceid=26, filesize=5.4 K 2024-12-11T22:58:58,808 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/b/dfb70d6837b04d21b8209d75eb194ad1 as hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/b/dfb70d6837b04d21b8209d75eb194ad1 2024-12-11T22:58:58,814 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/b/dfb70d6837b04d21b8209d75eb194ad1, entries=7, sequenceid=26, filesize=5.4 K 2024-12-11T22:58:58,815 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/.tmp/c/6f5c67a263044601b1f19ec9216c7c9f as hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/c/6f5c67a263044601b1f19ec9216c7c9f 2024-12-11T22:58:58,821 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/c/6f5c67a263044601b1f19ec9216c7c9f, entries=6, sequenceid=26, filesize=5.3 K 2024-12-11T22:58:58,821 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 228e6f3399ee99d920bd97c6b5418b49 in 146ms, sequenceid=26, compaction requested=false; wal=null 2024-12-11T22:58:58,822 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/0000000000000000026 2024-12-11T22:58:58,823 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,824 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,824 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:58,826 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,828 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsAfterAbortingFlush/228e6f3399ee99d920bd97c6b5418b49/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-11T22:58:58,829 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 228e6f3399ee99d920bd97c6b5418b49; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74761209, jitterRate=0.11402882635593414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:58,829 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 228e6f3399ee99d920bd97c6b5418b49 2024-12-11T22:58:58,830 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 228e6f3399ee99d920bd97c6b5418b49: Running coprocessor pre-open hook at 1733957938651Writing region info on filesystem at 1733957938651Initializing all the Stores at 1733957938652 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957938652Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957938653 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957938653Obtaining lock to block concurrent updates at 1733957938675 (+22 ms)Preparing flush snapshotting stores in 228e6f3399ee99d920bd97c6b5418b49 at 1733957938675Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733957938675Flushing stores of testReplayEditsAfterAbortingFlush,,1733957928935.228e6f3399ee99d920bd97c6b5418b49. at 1733957938675Flushing 228e6f3399ee99d920bd97c6b5418b49/a: creating writer at 1733957938676 (+1 ms)Flushing 228e6f3399ee99d920bd97c6b5418b49/a: appending metadata at 1733957938694 (+18 ms)Flushing 228e6f3399ee99d920bd97c6b5418b49/a: closing flushed file at 1733957938694Flushing 228e6f3399ee99d920bd97c6b5418b49/b: creating writer at 1733957938733 (+39 ms)Flushing 228e6f3399ee99d920bd97c6b5418b49/b: appending metadata at 1733957938750 (+17 ms)Flushing 228e6f3399ee99d920bd97c6b5418b49/b: closing flushed file at 1733957938750Flushing 228e6f3399ee99d920bd97c6b5418b49/c: creating writer at 1733957938770 (+20 ms)Flushing 228e6f3399ee99d920bd97c6b5418b49/c: appending metadata at 1733957938786 (+16 ms)Flushing 228e6f3399ee99d920bd97c6b5418b49/c: closing flushed file at 1733957938786Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45e62203: reopening flushed file at 1733957938800 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e5e2b0b: reopening flushed file at 1733957938808 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6629dcb: reopening flushed file at 1733957938814 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 228e6f3399ee99d920bd97c6b5418b49 in 146ms, sequenceid=26, compaction requested=false; wal=null at 1733957938821 (+7 ms)Cleaning up temporary data from old regions at 1733957938824 (+3 ms)Running coprocessor post-open hooks at 1733957938829 (+5 ms)Region opened successfully at 1733957938830 (+1 ms) 2024-12-11T22:58:58,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 421), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 125) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11432 (was 11440) 2024-12-11T22:58:58,854 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-11T22:58:58,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=143, ProcessCount=11, AvailableMemoryMB=11432 2024-12-11T22:58:58,875 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-11T22:58:58,899 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:58,901 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:58,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:58,905 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-94755819, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-94755819, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:58,927 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-94755819/hregion-94755819.1733957938905, exclude list is [], retry=0 2024-12-11T22:58:58,930 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:58,930 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:58,933 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:58,944 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-94755819/hregion-94755819.1733957938905 2024-12-11T22:58:58,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:58,944 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 62e883297e4182fff998c68f8630d552, NAME => 'testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:58,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741915_1093 (size=61) 2024-12-11T22:58:58,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741915_1093 (size=61) 2024-12-11T22:58:58,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741915_1093 (size=61) 2024-12-11T22:58:58,966 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:58,969 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:58,971 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62e883297e4182fff998c68f8630d552 columnFamilyName a 2024-12-11T22:58:58,971 DEBUG [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:58,972 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(327): Store=62e883297e4182fff998c68f8630d552/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:58,972 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:58,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:58,974 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:58,974 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:58,976 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:58,979 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:58,980 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62e883297e4182fff998c68f8630d552; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65650210, jitterRate=-0.02173563838005066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:58,981 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62e883297e4182fff998c68f8630d552: Writing region info on filesystem at 1733957938966Initializing all the Stores at 1733957938967 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957938967Cleaning up temporary data from old regions at 1733957938974 (+7 ms)Region opened successfully at 1733957938981 (+7 ms) 2024-12-11T22:58:58,981 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62e883297e4182fff998c68f8630d552, disabling compactions & flushes 2024-12-11T22:58:58,981 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:58,981 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:58,981 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. after waiting 0 ms 2024-12-11T22:58:58,981 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:58,982 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:58,982 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62e883297e4182fff998c68f8630d552: Waiting for close lock at 1733957938981Disabling compacts and flushes for region at 1733957938981Disabling writes for close at 1733957938981Writing region close event to WAL at 1733957938981Closed at 1733957938981 2024-12-11T22:58:58,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741914_1092 (size=93) 2024-12-11T22:58:58,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741914_1092 (size=93) 2024-12-11T22:58:58,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741914_1092 (size=93) 2024-12-11T22:58:58,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:58,999 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-94755819:(num 1733957938905) 2024-12-11T22:58:59,000 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:59,002 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:59,023 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003, exclude list is [], retry=0 2024-12-11T22:58:59,026 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:59,027 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:59,027 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:59,033 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003 2024-12-11T22:58:59,034 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:58:59,034 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 62e883297e4182fff998c68f8630d552, NAME => 'testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:59,034 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:59,034 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,034 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,036 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,037 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62e883297e4182fff998c68f8630d552 columnFamilyName a 2024-12-11T22:58:59,037 DEBUG [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,038 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(327): Store=62e883297e4182fff998c68f8630d552/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,038 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,038 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,040 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,040 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,040 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,042 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,044 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62e883297e4182fff998c68f8630d552; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69741074, jitterRate=0.03922298550605774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:59,045 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62e883297e4182fff998c68f8630d552: Writing region info on filesystem at 1733957939034Initializing all the Stores at 1733957939035 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939035Cleaning up temporary data from old regions at 1733957939040 (+5 ms)Region opened successfully at 1733957939045 (+5 ms) 2024-12-11T22:58:59,064 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62e883297e4182fff998c68f8630d552, disabling compactions & flushes 2024-12-11T22:58:59,064 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:59,064 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:59,064 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. after waiting 0 ms 2024-12-11T22:58:59,064 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:59,064 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:59,064 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. 2024-12-11T22:58:59,065 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62e883297e4182fff998c68f8630d552: Waiting for close lock at 1733957939063Disabling compacts and flushes for region at 1733957939063Disabling writes for close at 1733957939064 (+1 ms)Writing region close event to WAL at 1733957939064Closed at 1733957939064 2024-12-11T22:58:59,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741916_1094 (size=1623) 2024-12-11T22:58:59,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741916_1094 (size=1623) 2024-12-11T22:58:59,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741916_1094 (size=1623) 2024-12-11T22:58:59,100 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003, size=1.6 K (1623bytes) 2024-12-11T22:58:59,100 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003 2024-12-11T22:58:59,101 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003 after 0ms 2024-12-11T22:58:59,103 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003 took 4ms 2024-12-11T22:58:59,106 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003 so closing down 2024-12-11T22:58:59,106 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:58:59,108 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733957939003.temp 2024-12-11T22:58:59,111 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000003-wal.1733957939003.temp 2024-12-11T22:58:59,111 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:58:59,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741917_1095 (size=1623) 2024-12-11T22:58:59,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741917_1095 (size=1623) 2024-12-11T22:58:59,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741917_1095 (size=1623) 2024-12-11T22:58:59,122 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000003-wal.1733957939003.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-11T22:58:59,123 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000003-wal.1733957939003.temp to hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000012 2024-12-11T22:58:59,123 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-12-11T22:58:59,123 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003, journal: Splitting hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003, size=1.6 K (1623bytes) at 1733957939100Finishing writing output for hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003 so closing down at 1733957939106 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000003-wal.1733957939003.temp at 1733957939111 (+5 ms)3 split writer threads finished at 1733957939111Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000003-wal.1733957939003.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733957939122 (+11 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000003-wal.1733957939003.temp to hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000012 at 1733957939123 (+1 ms)Processed 10 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1733957939123 2024-12-11T22:58:59,125 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939003 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957939003 2024-12-11T22:58:59,126 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000012 2024-12-11T22:58:59,436 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:59,439 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:59,452 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939439, exclude list is [], retry=0 2024-12-11T22:58:59,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:59,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:59,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:59,457 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939439 2024-12-11T22:58:59,457 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 62e883297e4182fff998c68f8630d552, NAME => 'testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,460 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,462 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62e883297e4182fff998c68f8630d552 columnFamilyName a 2024-12-11T22:58:59,462 DEBUG [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,463 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(327): Store=62e883297e4182fff998c68f8630d552/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,463 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,464 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,465 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000012 2024-12-11T22:58:59,468 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000012 2024-12-11T22:58:59,468 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 62e883297e4182fff998c68f8630d552 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-11T22:58:59,487 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/.tmp/a/2b3321ff0de04c44a828045bef1369b4 is 79, key is testDatalossWhenInputError/a:x0/1733957939045/Put/seqid=0 2024-12-11T22:58:59,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741919_1097 (size=5808) 2024-12-11T22:58:59,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741919_1097 (size=5808) 2024-12-11T22:58:59,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741919_1097 (size=5808) 2024-12-11T22:58:59,494 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/.tmp/a/2b3321ff0de04c44a828045bef1369b4 2024-12-11T22:58:59,502 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/.tmp/a/2b3321ff0de04c44a828045bef1369b4 as hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/a/2b3321ff0de04c44a828045bef1369b4 2024-12-11T22:58:59,509 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/a/2b3321ff0de04c44a828045bef1369b4, entries=10, sequenceid=12, filesize=5.7 K 2024-12-11T22:58:59,509 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 62e883297e4182fff998c68f8630d552 in 41ms, sequenceid=12, compaction requested=false; wal=null 2024-12-11T22:58:59,510 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/0000000000000000012 2024-12-11T22:58:59,511 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,511 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,514 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,516 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-11T22:58:59,516 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62e883297e4182fff998c68f8630d552; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61918723, jitterRate=-0.07733912765979767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:59,517 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62e883297e4182fff998c68f8630d552: Writing region info on filesystem at 1733957939457Initializing all the Stores at 1733957939460 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939460Obtaining lock to block concurrent updates at 1733957939468 (+8 ms)Preparing flush snapshotting stores in 62e883297e4182fff998c68f8630d552 at 1733957939468Finished memstore snapshotting testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733957939468Flushing stores of testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552. at 1733957939468Flushing 62e883297e4182fff998c68f8630d552/a: creating writer at 1733957939469 (+1 ms)Flushing 62e883297e4182fff998c68f8630d552/a: appending metadata at 1733957939486 (+17 ms)Flushing 62e883297e4182fff998c68f8630d552/a: closing flushed file at 1733957939487 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8a7d175: reopening flushed file at 1733957939500 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 62e883297e4182fff998c68f8630d552 in 41ms, sequenceid=12, compaction requested=false; wal=null at 1733957939509 (+9 ms)Cleaning up temporary data from old regions at 1733957939511 (+2 ms)Region opened successfully at 1733957939517 (+6 ms) 2024-12-11T22:58:59,520 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 62e883297e4182fff998c68f8630d552, NAME => 'testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:59,520 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733957938900.62e883297e4182fff998c68f8630d552.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:59,520 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,520 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,522 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,522 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62e883297e4182fff998c68f8630d552 columnFamilyName a 2024-12-11T22:58:59,522 DEBUG [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,528 DEBUG [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/a/2b3321ff0de04c44a828045bef1369b4 2024-12-11T22:58:59,528 INFO [StoreOpener-62e883297e4182fff998c68f8630d552-1 {}] regionserver.HStore(327): Store=62e883297e4182fff998c68f8630d552/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,528 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,529 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,530 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,530 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,530 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,531 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62e883297e4182fff998c68f8630d552 2024-12-11T22:58:59,533 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testDatalossWhenInputError/62e883297e4182fff998c68f8630d552/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-11T22:58:59,534 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62e883297e4182fff998c68f8630d552; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65263875, jitterRate=-0.027492478489875793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T22:58:59,534 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62e883297e4182fff998c68f8630d552: Writing region info on filesystem at 1733957939520Initializing all the Stores at 1733957939521 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939521Cleaning up temporary data from old regions at 1733957939530 (+9 ms)Region opened successfully at 1733957939534 (+4 ms) 2024-12-11T22:58:59,552 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=430 (was 420) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:52296 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:50402 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:46272 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:46348 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:52232 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:50496 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1204600519-172.17.0.3-1733957906467:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 143) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11415 (was 11432) 2024-12-11T22:58:59,552 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-11T22:58:59,565 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=11414 2024-12-11T22:58:59,565 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-11T22:58:59,580 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:59,583 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:58:59,583 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:58:59,586 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39882615, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-39882615, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:59,599 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39882615/hregion-39882615.1733957939586, exclude list is [], retry=0 2024-12-11T22:58:59,601 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:59,601 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:59,602 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:59,603 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39882615/hregion-39882615.1733957939586 2024-12-11T22:58:59,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745)] 2024-12-11T22:58:59,604 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0d791414f2aeea8946f02ef41668e0eb, NAME => 'testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:58:59,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741921_1099 (size=63) 2024-12-11T22:58:59,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741921_1099 (size=63) 2024-12-11T22:58:59,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741921_1099 (size=63) 2024-12-11T22:58:59,612 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:59,613 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,614 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName a 2024-12-11T22:58:59,614 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,615 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,615 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,616 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName b 2024-12-11T22:58:59,616 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,616 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,616 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,617 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName c 2024-12-11T22:58:59,617 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,618 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,618 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,619 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,619 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,620 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,620 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,621 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:59,622 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,624 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:58:59,625 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d791414f2aeea8946f02ef41668e0eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64208658, jitterRate=-0.043216437101364136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:59,626 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d791414f2aeea8946f02ef41668e0eb: Writing region info on filesystem at 1733957939612Initializing all the Stores at 1733957939613 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939613Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939613Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939613Cleaning up temporary data from old regions at 1733957939620 (+7 ms)Region opened successfully at 1733957939626 (+6 ms) 2024-12-11T22:58:59,626 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d791414f2aeea8946f02ef41668e0eb, disabling compactions & flushes 2024-12-11T22:58:59,626 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:58:59,626 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:58:59,626 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. after waiting 0 ms 2024-12-11T22:58:59,626 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:58:59,626 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:58:59,626 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d791414f2aeea8946f02ef41668e0eb: Waiting for close lock at 1733957939626Disabling compacts and flushes for region at 1733957939626Disabling writes for close at 1733957939626Writing region close event to WAL at 1733957939626Closed at 1733957939626 2024-12-11T22:58:59,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741920_1098 (size=93) 2024-12-11T22:58:59,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741920_1098 (size=93) 2024-12-11T22:58:59,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741920_1098 (size=93) 2024-12-11T22:58:59,631 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:58:59,631 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39882615:(num 1733957939586) 2024-12-11T22:58:59,631 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:58:59,633 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:58:59,646 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633, exclude list is [], retry=0 2024-12-11T22:58:59,648 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:58:59,649 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:58:59,649 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:58:59,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 2024-12-11T22:58:59,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:58:59,653 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d791414f2aeea8946f02ef41668e0eb, NAME => 'testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:58:59,653 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:58:59,653 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,653 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,656 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,657 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName a 2024-12-11T22:58:59,657 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,658 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,658 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,659 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName b 2024-12-11T22:58:59,659 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,659 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,659 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,660 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName c 2024-12-11T22:58:59,660 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:58:59,660 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:58:59,660 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,661 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,662 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,663 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,663 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,663 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:58:59,665 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:58:59,665 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d791414f2aeea8946f02ef41668e0eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71380863, jitterRate=0.063657745718956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:58:59,666 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d791414f2aeea8946f02ef41668e0eb: Writing region info on filesystem at 1733957939653Initializing all the Stores at 1733957939654 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939654Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939656 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957939656Cleaning up temporary data from old regions at 1733957939663 (+7 ms)Region opened successfully at 1733957939666 (+3 ms) 2024-12-11T22:58:59,670 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733957939669/Put/seqid=0 2024-12-11T22:58:59,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741923_1101 (size=4875) 2024-12-11T22:58:59,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741923_1101 (size=4875) 2024-12-11T22:58:59,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741923_1101 (size=4875) 2024-12-11T22:58:59,682 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733957939681/Put/seqid=0 2024-12-11T22:58:59,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741924_1102 (size=4875) 2024-12-11T22:58:59,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741924_1102 (size=4875) 2024-12-11T22:58:59,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741924_1102 (size=4875) 2024-12-11T22:58:59,691 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733957939691/Put/seqid=0 2024-12-11T22:58:59,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741925_1103 (size=4875) 2024-12-11T22:58:59,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741925_1103 (size=4875) 2024-12-11T22:58:59,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741925_1103 (size=4875) 2024-12-11T22:58:59,698 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 0d791414f2aeea8946f02ef41668e0eb/a 2024-12-11T22:58:59,701 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-11T22:58:59,701 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-11T22:58:59,701 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 0d791414f2aeea8946f02ef41668e0eb/a 2024-12-11T22:58:59,704 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-11T22:58:59,704 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-11T22:58:59,705 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 0d791414f2aeea8946f02ef41668e0eb/a 2024-12-11T22:58:59,707 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-11T22:58:59,707 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-11T22:58:59,707 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0d791414f2aeea8946f02ef41668e0eb 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-11T22:58:59,723 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/.tmp/a/7ac5b1aa16524c73aeeb819717d0bdd0 is 55, key is testCompactedBulkLoadedFiles/a:a/1733957939666/Put/seqid=0 2024-12-11T22:58:59,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741926_1104 (size=5107) 2024-12-11T22:58:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741926_1104 (size=5107) 2024-12-11T22:58:59,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741926_1104 (size=5107) 2024-12-11T22:58:59,730 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/.tmp/a/7ac5b1aa16524c73aeeb819717d0bdd0 2024-12-11T22:58:59,736 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/.tmp/a/7ac5b1aa16524c73aeeb819717d0bdd0 as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0 2024-12-11T22:58:59,741 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0, entries=1, sequenceid=4, filesize=5.0 K 2024-12-11T22:58:59,742 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 0d791414f2aeea8946f02ef41668e0eb in 35ms, sequenceid=4, compaction requested=false 2024-12-11T22:58:59,742 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0d791414f2aeea8946f02ef41668e0eb: 2024-12-11T22:58:59,743 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_ 2024-12-11T22:58:59,745 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_ 2024-12-11T22:58:59,745 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_ 2024-12-11T22:58:59,746 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile0 into 0d791414f2aeea8946f02ef41668e0eb/a as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_ - updating store file list. 2024-12-11T22:58:59,750 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 433dd31efc7a455987c5e4c92d827fd2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-11T22:58:59,750 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_ into 0d791414f2aeea8946f02ef41668e0eb/a 2024-12-11T22:58:59,750 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile0 into 0d791414f2aeea8946f02ef41668e0eb/a (new location: hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_) 2024-12-11T22:58:59,751 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile1 into 0d791414f2aeea8946f02ef41668e0eb/a as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_ - updating store file list. 2024-12-11T22:58:59,755 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3d9056b0a2604770bfaa368f7be8a269_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-11T22:58:59,755 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_ into 0d791414f2aeea8946f02ef41668e0eb/a 2024-12-11T22:58:59,755 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile1 into 0d791414f2aeea8946f02ef41668e0eb/a (new location: hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_) 2024-12-11T22:58:59,756 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile2 into 0d791414f2aeea8946f02ef41668e0eb/a as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_ - updating store file list. 2024-12-11T22:58:59,760 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8d0eb5429bab4e9498b35366c37b9952_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-11T22:58:59,760 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_ into 0d791414f2aeea8946f02ef41668e0eb/a 2024-12-11T22:58:59,760 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38049/hbase/testCompactedBulkLoadedFiles/hfile2 into 0d791414f2aeea8946f02ef41668e0eb/a (new location: hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_) 2024-12-11T22:58:59,767 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T22:58:59,767 DEBUG [Time-limited test {}] regionserver.HStore(1541): 0d791414f2aeea8946f02ef41668e0eb/a is initiating major compaction (all files) 2024-12-11T22:58:59,767 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 0d791414f2aeea8946f02ef41668e0eb/a in testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:58:59,767 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_] into tmpdir=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/.tmp, totalSize=19.3 K 2024-12-11T22:58:59,768 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7ac5b1aa16524c73aeeb819717d0bdd0, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733957939666 2024-12-11T22:58:59,768 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 433dd31efc7a455987c5e4c92d827fd2_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-11T22:58:59,769 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3d9056b0a2604770bfaa368f7be8a269_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-11T22:58:59,769 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8d0eb5429bab4e9498b35366c37b9952_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-11T22:58:59,780 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/.tmp/a/38979699a8734433aa5992106eb0faf3 is 55, key is testCompactedBulkLoadedFiles/a:a/1733957939666/Put/seqid=0 2024-12-11T22:58:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741927_1105 (size=6154) 2024-12-11T22:58:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741927_1105 (size=6154) 2024-12-11T22:58:59,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741927_1105 (size=6154) 2024-12-11T22:58:59,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/.tmp/a/38979699a8734433aa5992106eb0faf3 as hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/38979699a8734433aa5992106eb0faf3 2024-12-11T22:58:59,802 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 0d791414f2aeea8946f02ef41668e0eb/a of 0d791414f2aeea8946f02ef41668e0eb into 38979699a8734433aa5992106eb0faf3(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T22:58:59,802 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 0d791414f2aeea8946f02ef41668e0eb: 2024-12-11T22:58:59,802 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-11T22:58:59,802 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-11T22:58:59,835 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633, size=0 (0bytes) 2024-12-11T22:58:59,835 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 might be still open, length is 0 2024-12-11T22:58:59,835 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 2024-12-11T22:58:59,835 WARN [IPC Server handler 1 on default port 38049 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-11T22:58:59,835 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 after 0ms 2024-12-11T22:58:59,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-11T22:58:59,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-11T22:58:59,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-11T22:58:59,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-11T22:59:01,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:52334 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43249:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52334 dst: /127.0.0.1:43249 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43249 remote=/127.0.0.1:52334]. Total timeout mills is 60000, 58412 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:01,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:50514 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50514 dst: /127.0.0.1:39291 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:01,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:46392 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37177:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46392 dst: /127.0.0.1:37177 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:01,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741922_1106 (size=1659) 2024-12-11T22:59:01,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741922_1106 (size=1659) 2024-12-11T22:59:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741922_1106 (size=1659) 2024-12-11T22:59:03,836 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 after 4001ms 2024-12-11T22:59:03,839 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 took 4005ms 2024-12-11T22:59:03,841 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633; continuing. 2024-12-11T22:59:03,841 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 so closing down 2024-12-11T22:59:03,841 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:59:03,843 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733957939633.temp 2024-12-11T22:59:03,844 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000003-wal.1733957939633.temp 2024-12-11T22:59:03,844 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:59:03,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741928_1107 (size=698) 2024-12-11T22:59:03,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741928_1107 (size=698) 2024-12-11T22:59:03,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741928_1107 (size=698) 2024-12-11T22:59:03,858 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000003-wal.1733957939633.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-11T22:59:03,860 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000003-wal.1733957939633.temp to hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000008 2024-12-11T22:59:03,860 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633, size=0, length=0, corrupted=false, cancelled=false 2024-12-11T22:59:03,860 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633, journal: Splitting hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633, size=0 (0bytes) at 1733957939835Finishing writing output for hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 so closing down at 1733957943841 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000003-wal.1733957939633.temp at 1733957943844 (+3 ms)3 split writer threads finished at 1733957943844Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000003-wal.1733957939633.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733957943858 (+14 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000003-wal.1733957939633.temp to hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000008 at 1733957943860 (+2 ms)Processed 5 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633, size=0, length=0, corrupted=false, cancelled=false at 1733957943860 2024-12-11T22:59:03,862 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957939633 2024-12-11T22:59:03,863 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000008 2024-12-11T22:59:03,863 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:59:03,864 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:59:03,878 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957943865, exclude list is [], retry=0 2024-12-11T22:59:03,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:59:03,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:59:03,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:59:03,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957943865 2024-12-11T22:59:03,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:59:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d791414f2aeea8946f02ef41668e0eb, NAME => 'testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:59:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:59:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,884 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,888 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,889 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName a 2024-12-11T22:59:03,889 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:03,896 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/38979699a8734433aa5992106eb0faf3 2024-12-11T22:59:03,900 DEBUG [StoreFileOpener-0d791414f2aeea8946f02ef41668e0eb-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3d9056b0a2604770bfaa368f7be8a269_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-11T22:59:03,900 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_ 2024-12-11T22:59:03,903 DEBUG [StoreFileOpener-0d791414f2aeea8946f02ef41668e0eb-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 433dd31efc7a455987c5e4c92d827fd2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-11T22:59:03,903 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_ 2024-12-11T22:59:03,907 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0 2024-12-11T22:59:03,910 DEBUG [StoreFileOpener-0d791414f2aeea8946f02ef41668e0eb-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8d0eb5429bab4e9498b35366c37b9952_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-11T22:59:03,910 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_ 2024-12-11T22:59:03,910 WARN [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-12-11T22:59:03,911 WARN [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-12-11T22:59:03,911 WARN [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-12-11T22:59:03,911 WARN [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5e4c8cfb 2024-12-11T22:59:03,911 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_] to archive 2024-12-11T22:59:03,912 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T22:59:03,914 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_ to hdfs://localhost:38049/hbase/archive/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/3d9056b0a2604770bfaa368f7be8a269_SeqId_4_ 2024-12-11T22:59:03,916 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_ to hdfs://localhost:38049/hbase/archive/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/433dd31efc7a455987c5e4c92d827fd2_SeqId_4_ 2024-12-11T22:59:03,917 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0 to hdfs://localhost:38049/hbase/archive/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/7ac5b1aa16524c73aeeb819717d0bdd0 2024-12-11T22:59:03,917 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_ to hdfs://localhost:38049/hbase/archive/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/a/8d0eb5429bab4e9498b35366c37b9952_SeqId_4_ 2024-12-11T22:59:03,917 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:03,917 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,918 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName b 2024-12-11T22:59:03,918 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:03,918 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:03,918 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,919 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d791414f2aeea8946f02ef41668e0eb columnFamilyName c 2024-12-11T22:59:03,919 DEBUG [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:03,920 INFO [StoreOpener-0d791414f2aeea8946f02ef41668e0eb-1 {}] regionserver.HStore(327): Store=0d791414f2aeea8946f02ef41668e0eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:03,920 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,920 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,922 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,922 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000008 2024-12-11T22:59:03,926 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 0d791414f2aeea8946f02ef41668e0eb : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "0d791414f2aeea8946f02ef41668e0eb" family_name: "a" compaction_input: "7ac5b1aa16524c73aeeb819717d0bdd0" compaction_input: "433dd31efc7a455987c5e4c92d827fd2_SeqId_4_" compaction_input: "3d9056b0a2604770bfaa368f7be8a269_SeqId_4_" compaction_input: "8d0eb5429bab4e9498b35366c37b9952_SeqId_4_" compaction_output: "38979699a8734433aa5992106eb0faf3" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-11T22:59:03,926 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-11T22:59:03,926 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000008 2024-12-11T22:59:03,927 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/0000000000000000008 2024-12-11T22:59:03,928 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,928 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,929 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:59:03,930 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0d791414f2aeea8946f02ef41668e0eb 2024-12-11T22:59:03,933 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testCompactedBulkLoadedFiles/0d791414f2aeea8946f02ef41668e0eb/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-11T22:59:03,934 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0d791414f2aeea8946f02ef41668e0eb; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60146426, jitterRate=-0.10374841094017029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:59:03,934 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0d791414f2aeea8946f02ef41668e0eb: Writing region info on filesystem at 1733957943884Initializing all the Stores at 1733957943885 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957943885Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957943888 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957943888Cleaning up temporary data from old regions at 1733957943928 (+40 ms)Region opened successfully at 1733957943934 (+6 ms) 2024-12-11T22:59:03,936 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0d791414f2aeea8946f02ef41668e0eb, disabling compactions & flushes 2024-12-11T22:59:03,936 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:59:03,937 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:59:03,937 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. after waiting 0 ms 2024-12-11T22:59:03,937 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:59:03,938 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733957939581.0d791414f2aeea8946f02ef41668e0eb. 2024-12-11T22:59:03,938 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0d791414f2aeea8946f02ef41668e0eb: Waiting for close lock at 1733957943936Disabling compacts and flushes for region at 1733957943936Disabling writes for close at 1733957943937 (+1 ms)Writing region close event to WAL at 1733957943937Closed at 1733957943938 (+1 ms) 2024-12-11T22:59:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741929_1108 (size=93) 2024-12-11T22:59:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741929_1108 (size=93) 2024-12-11T22:59:03,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741929_1108 (size=93) 2024-12-11T22:59:03,943 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:59:03,943 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733957943865) 2024-12-11T22:59:03,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=446 (was 430) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1796836718_22 at /127.0.0.1:50654 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:38049 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1796836718_22 at /127.0.0.1:46446 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (80400340) connection to localhost/127.0.0.1:38049 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 228), ProcessCount=11 (was 11), AvailableMemoryMB=12428 (was 11414) - AvailableMemoryMB LEAK? - 2024-12-11T22:59:03,960 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-11T22:59:03,975 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=446, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=12426 2024-12-11T22:59:03,975 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-11T22:59:03,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:59:04,001 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T22:59:04,002 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T22:59:04,005 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73315301, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/hregion-73315301, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:59:04,019 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73315301/hregion-73315301.1733957944005, exclude list is [], retry=0 2024-12-11T22:59:04,022 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:59:04,022 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:59:04,023 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:59:04,025 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73315301/hregion-73315301.1733957944005 2024-12-11T22:59:04,026 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:59:04,027 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3329c68d438c53ad4de408bf72dbbcdf, NAME => 'testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38049/hbase 2024-12-11T22:59:04,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741931_1110 (size=67) 2024-12-11T22:59:04,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741931_1110 (size=67) 2024-12-11T22:59:04,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741931_1110 (size=67) 2024-12-11T22:59:04,042 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:59:04,046 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,048 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName a 2024-12-11T22:59:04,048 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,048 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,049 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,050 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName b 2024-12-11T22:59:04,050 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,051 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,051 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,052 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName c 2024-12-11T22:59:04,052 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,053 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,053 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,054 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,054 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,055 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,055 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,056 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:59:04,065 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,068 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T22:59:04,069 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3329c68d438c53ad4de408bf72dbbcdf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62979628, jitterRate=-0.06153041124343872}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:59:04,069 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3329c68d438c53ad4de408bf72dbbcdf: Writing region info on filesystem at 1733957944043Initializing all the Stores at 1733957944045 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944046 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944046Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944046Cleaning up temporary data from old regions at 1733957944055 (+9 ms)Region opened successfully at 1733957944069 (+14 ms) 2024-12-11T22:59:04,069 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3329c68d438c53ad4de408bf72dbbcdf, disabling compactions & flushes 2024-12-11T22:59:04,069 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,069 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,069 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. after waiting 0 ms 2024-12-11T22:59:04,069 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,070 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,070 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3329c68d438c53ad4de408bf72dbbcdf: Waiting for close lock at 1733957944069Disabling compacts and flushes for region at 1733957944069Disabling writes for close at 1733957944069Writing region close event to WAL at 1733957944070 (+1 ms)Closed at 1733957944070 2024-12-11T22:59:04,071 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T22:59:04,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741930_1109 (size=93) 2024-12-11T22:59:04,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741930_1109 (size=93) 2024-12-11T22:59:04,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741930_1109 (size=93) 2024-12-11T22:59:04,078 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:59:04,078 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73315301:(num 1733957944005) 2024-12-11T22:59:04,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:59:04,080 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:59:04,102 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081, exclude list is [], retry=0 2024-12-11T22:59:04,105 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:59:04,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:59:04,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:59:04,114 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081 2024-12-11T22:59:04,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:59:04,117 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3329c68d438c53ad4de408bf72dbbcdf, NAME => 'testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:59:04,117 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:59:04,117 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,117 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,121 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,122 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName a 2024-12-11T22:59:04,122 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,122 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,122 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,123 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName b 2024-12-11T22:59:04,123 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,124 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,124 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,126 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName c 2024-12-11T22:59:04,126 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,129 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,129 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,130 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,131 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,132 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,132 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,132 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:59:04,134 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,134 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3329c68d438c53ad4de408bf72dbbcdf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74486301, jitterRate=0.10993237793445587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:59:04,135 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3329c68d438c53ad4de408bf72dbbcdf: Writing region info on filesystem at 1733957944117Initializing all the Stores at 1733957944118 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944118Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944120 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944120Cleaning up temporary data from old regions at 1733957944132 (+12 ms)Region opened successfully at 1733957944135 (+3 ms) 2024-12-11T22:59:04,143 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3329c68d438c53ad4de408bf72dbbcdf 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-11T22:59:04,162 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/a/19c51233195748e5892c3e47226935d3 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733957944135/Put/seqid=0 2024-12-11T22:59:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741933_1112 (size=5958) 2024-12-11T22:59:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741933_1112 (size=5958) 2024-12-11T22:59:04,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741933_1112 (size=5958) 2024-12-11T22:59:04,181 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/a/19c51233195748e5892c3e47226935d3 2024-12-11T22:59:04,187 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/a/19c51233195748e5892c3e47226935d3 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/a/19c51233195748e5892c3e47226935d3 2024-12-11T22:59:04,193 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/a/19c51233195748e5892c3e47226935d3, entries=10, sequenceid=13, filesize=5.8 K 2024-12-11T22:59:04,194 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 3329c68d438c53ad4de408bf72dbbcdf in 51ms, sequenceid=13, compaction requested=false 2024-12-11T22:59:04,194 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3329c68d438c53ad4de408bf72dbbcdf: 2024-12-11T22:59:04,217 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3329c68d438c53ad4de408bf72dbbcdf, disabling compactions & flushes 2024-12-11T22:59:04,218 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,218 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,218 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. after waiting 0 ms 2024-12-11T22:59:04,218 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,221 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,221 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:04,221 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3329c68d438c53ad4de408bf72dbbcdf: Waiting for close lock at 1733957944217Disabling compacts and flushes for region at 1733957944217Disabling writes for close at 1733957944218 (+1 ms)Writing region close event to WAL at 1733957944220 (+2 ms)Closed at 1733957944221 (+1 ms) 2024-12-11T22:59:04,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741932_1111 (size=5869) 2024-12-11T22:59:04,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741932_1111 (size=5869) 2024-12-11T22:59:04,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741932_1111 (size=5869) 2024-12-11T22:59:04,245 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081, size=5.7 K (5869bytes) 2024-12-11T22:59:04,245 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081 2024-12-11T22:59:04,245 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081 after 0ms 2024-12-11T22:59:04,247 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081 took 3ms 2024-12-11T22:59:04,250 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081 so closing down 2024-12-11T22:59:04,251 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:59:04,252 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733957944081.temp 2024-12-11T22:59:04,255 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000003-wal.1733957944081.temp 2024-12-11T22:59:04,255 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:59:04,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741934_1113 (size=5223) 2024-12-11T22:59:04,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741934_1113 (size=5223) 2024-12-11T22:59:04,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741934_1113 (size=5223) 2024-12-11T22:59:04,269 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000003-wal.1733957944081.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-11T22:59:04,271 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000003-wal.1733957944081.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000035 2024-12-11T22:59:04,271 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-12-11T22:59:04,271 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081, journal: Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081, size=5.7 K (5869bytes) at 1733957944245Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081 so closing down at 1733957944250 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000003-wal.1733957944081.temp at 1733957944255 (+5 ms)3 split writer threads finished at 1733957944255Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000003-wal.1733957944081.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733957944269 (+14 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000003-wal.1733957944081.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000035 at 1733957944271 (+2 ms)Processed 32 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1733957944271 2024-12-11T22:59:04,273 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944081 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957944081 2024-12-11T22:59:04,274 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000035 2024-12-11T22:59:04,274 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:59:04,275 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:59:04,293 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276, exclude list is [], retry=0 2024-12-11T22:59:04,296 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:59:04,296 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:59:04,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:59:04,298 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 2024-12-11T22:59:04,300 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895),(127.0.0.1/127.0.0.1:35233:35233)] 2024-12-11T22:59:04,301 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3329c68d438c53ad4de408bf72dbbcdf, NAME => 'testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf.', STARTKEY => '', ENDKEY => ''} 2024-12-11T22:59:04,301 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:59:04,301 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,301 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,303 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,303 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName a 2024-12-11T22:59:04,303 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,310 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/a/19c51233195748e5892c3e47226935d3 2024-12-11T22:59:04,310 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,310 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,311 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName b 2024-12-11T22:59:04,312 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,312 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,312 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,313 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName c 2024-12-11T22:59:04,313 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:04,314 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:04,314 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,321 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000035 2024-12-11T22:59:04,325 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000035 2024-12-11T22:59:04,325 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3329c68d438c53ad4de408bf72dbbcdf 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-11T22:59:04,350 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/b/1c6f8ff6865f427f8264cb7e517dd7d3 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733957944194/Put/seqid=0 2024-12-11T22:59:04,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741936_1115 (size=5958) 2024-12-11T22:59:04,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741936_1115 (size=5958) 2024-12-11T22:59:04,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741936_1115 (size=5958) 2024-12-11T22:59:04,365 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/b/1c6f8ff6865f427f8264cb7e517dd7d3 2024-12-11T22:59:04,387 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/c/55712c98f2e841e695ea910d00b07ef9 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733957944203/Put/seqid=0 2024-12-11T22:59:04,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741937_1116 (size=5958) 2024-12-11T22:59:04,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741937_1116 (size=5958) 2024-12-11T22:59:04,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741937_1116 (size=5958) 2024-12-11T22:59:04,399 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/c/55712c98f2e841e695ea910d00b07ef9 2024-12-11T22:59:04,405 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/b/1c6f8ff6865f427f8264cb7e517dd7d3 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/b/1c6f8ff6865f427f8264cb7e517dd7d3 2024-12-11T22:59:04,410 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/b/1c6f8ff6865f427f8264cb7e517dd7d3, entries=10, sequenceid=35, filesize=5.8 K 2024-12-11T22:59:04,411 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/c/55712c98f2e841e695ea910d00b07ef9 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/c/55712c98f2e841e695ea910d00b07ef9 2024-12-11T22:59:04,416 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/c/55712c98f2e841e695ea910d00b07ef9, entries=10, sequenceid=35, filesize=5.8 K 2024-12-11T22:59:04,416 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 3329c68d438c53ad4de408bf72dbbcdf in 91ms, sequenceid=35, compaction requested=false; wal=null 2024-12-11T22:59:04,417 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000035 2024-12-11T22:59:04,417 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,418 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,418 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:59:04,420 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:04,424 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-11T22:59:04,425 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3329c68d438c53ad4de408bf72dbbcdf; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74346730, jitterRate=0.10785260796546936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:59:04,425 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3329c68d438c53ad4de408bf72dbbcdf: Writing region info on filesystem at 1733957944301Initializing all the Stores at 1733957944302 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944302Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944303 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957944303Obtaining lock to block concurrent updates at 1733957944325 (+22 ms)Preparing flush snapshotting stores in 3329c68d438c53ad4de408bf72dbbcdf at 1733957944325Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733957944325Flushing stores of testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. at 1733957944325Flushing 3329c68d438c53ad4de408bf72dbbcdf/b: creating writer at 1733957944325Flushing 3329c68d438c53ad4de408bf72dbbcdf/b: appending metadata at 1733957944349 (+24 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/b: closing flushed file at 1733957944349Flushing 3329c68d438c53ad4de408bf72dbbcdf/c: creating writer at 1733957944371 (+22 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/c: appending metadata at 1733957944386 (+15 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/c: closing flushed file at 1733957944386Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10c505b8: reopening flushed file at 1733957944404 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10eefe5f: reopening flushed file at 1733957944410 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 3329c68d438c53ad4de408bf72dbbcdf in 91ms, sequenceid=35, compaction requested=false; wal=null at 1733957944416 (+6 ms)Cleaning up temporary data from old regions at 1733957944418 (+2 ms)Region opened successfully at 1733957944425 (+7 ms) 2024-12-11T22:59:04,521 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276, size=0 (0bytes) 2024-12-11T22:59:04,521 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 might be still open, length is 0 2024-12-11T22:59:04,521 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 2024-12-11T22:59:04,522 WARN [IPC Server handler 3 on default port 38049 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-11T22:59:04,522 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 after 1ms 2024-12-11T22:59:07,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:46504 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37177:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46504 dst: /127.0.0.1:37177 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37177 remote=/127.0.0.1:46504]. Total timeout mills is 60000, 57080 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:07,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:52450 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43249:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52450 dst: /127.0.0.1:43249 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:07,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_866042119_22 at /127.0.0.1:50700 [Receiving block BP-1204600519-172.17.0.3-1733957906467:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39291:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50700 dst: /127.0.0.1:39291 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:07,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741935_1117 (size=5215) 2024-12-11T22:59:07,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741935_1117 (size=5215) 2024-12-11T22:59:08,525 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 after 4003ms 2024-12-11T22:59:08,529 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 took 4007ms 2024-12-11T22:59:08,540 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276; continuing. 2024-12-11T22:59:08,540 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 so closing down 2024-12-11T22:59:08,540 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-11T22:59:08,542 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733957944276.temp 2024-12-11T22:59:08,543 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000037-wal.1733957944276.temp 2024-12-11T22:59:08,543 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-11T22:59:08,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741938_1118 (size=5223) 2024-12-11T22:59:08,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741938_1118 (size=5223) 2024-12-11T22:59:08,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741938_1118 (size=5223) 2024-12-11T22:59:08,553 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000037-wal.1733957944276.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-11T22:59:08,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000037-wal.1733957944276.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000066 2024-12-11T22:59:08,555 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276, size=0, length=0, corrupted=false, cancelled=false 2024-12-11T22:59:08,555 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276, journal: Splitting hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276, size=0 (0bytes) at 1733957944521Finishing writing output for hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 so closing down at 1733957948540 (+4019 ms)Creating recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000037-wal.1733957944276.temp at 1733957948543 (+3 ms)3 split writer threads finished at 1733957948543Closed recovered edits writer path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000037-wal.1733957944276.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733957948553 (+10 ms)Rename recovered edits hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000037-wal.1733957944276.temp to hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000066 at 1733957948555 (+2 ms)Processed 30 edits across 1 Regions in 20 ms; skipped=0; WAL=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276, size=0, length=0, corrupted=false, cancelled=false at 1733957948555 2024-12-11T22:59:08,557 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 to hdfs://localhost:38049/hbase/oldWALs/wal.1733957944276 2024-12-11T22:59:08,557 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000066 2024-12-11T22:59:08,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-11T22:59:08,559 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38049/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998, archiveDir=hdfs://localhost:38049/hbase/oldWALs, maxLogs=32 2024-12-11T22:59:08,573 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957948560, exclude list is [], retry=0 2024-12-11T22:59:08,575 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39291,DS-15b6e92b-caf8-49e9-bd73-26e8dd2949d4,DISK] 2024-12-11T22:59:08,576 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37177,DS-6c1894a9-6b42-4116-a0c8-df45eb92e533,DISK] 2024-12-11T22:59:08,576 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43249,DS-af6d2772-c851-4c16-96c2-835a220a12da,DISK] 2024-12-11T22:59:08,577 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957948560 2024-12-11T22:59:08,578 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35233:35233),(127.0.0.1/127.0.0.1:43745:43745),(127.0.0.1/127.0.0.1:40895:40895)] 2024-12-11T22:59:08,578 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T22:59:08,580 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,580 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName a 2024-12-11T22:59:08,580 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:08,588 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/a/19c51233195748e5892c3e47226935d3 2024-12-11T22:59:08,588 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:08,588 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,589 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName b 2024-12-11T22:59:08,589 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:08,593 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/b/1c6f8ff6865f427f8264cb7e517dd7d3 2024-12-11T22:59:08,593 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:08,593 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,594 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3329c68d438c53ad4de408bf72dbbcdf columnFamilyName c 2024-12-11T22:59:08,594 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T22:59:08,598 DEBUG [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/c/55712c98f2e841e695ea910d00b07ef9 2024-12-11T22:59:08,598 INFO [StoreOpener-3329c68d438c53ad4de408bf72dbbcdf-1 {}] regionserver.HStore(327): Store=3329c68d438c53ad4de408bf72dbbcdf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T22:59:08,598 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,599 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,600 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,601 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000066 2024-12-11T22:59:08,607 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000066 2024-12-11T22:59:08,607 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3329c68d438c53ad4de408bf72dbbcdf 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-11T22:59:08,622 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/a/bb4a552face942e38e982f8301b3e93a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733957944433/Put/seqid=0 2024-12-11T22:59:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741940_1120 (size=5958) 2024-12-11T22:59:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741940_1120 (size=5958) 2024-12-11T22:59:08,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741940_1120 (size=5958) 2024-12-11T22:59:08,629 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/a/bb4a552face942e38e982f8301b3e93a 2024-12-11T22:59:08,648 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/b/e5284bda80924697b9790626b29a6101 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733957944445/Put/seqid=0 2024-12-11T22:59:08,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741941_1121 (size=5958) 2024-12-11T22:59:08,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741941_1121 (size=5958) 2024-12-11T22:59:08,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741941_1121 (size=5958) 2024-12-11T22:59:08,655 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/b/e5284bda80924697b9790626b29a6101 2024-12-11T22:59:08,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/c/30483c3e175944189acb28a3f64f03c5 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733957944460/Put/seqid=0 2024-12-11T22:59:08,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741942_1122 (size=5958) 2024-12-11T22:59:08,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741942_1122 (size=5958) 2024-12-11T22:59:08,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741942_1122 (size=5958) 2024-12-11T22:59:08,682 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/c/30483c3e175944189acb28a3f64f03c5 2024-12-11T22:59:08,687 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/a/bb4a552face942e38e982f8301b3e93a as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/a/bb4a552face942e38e982f8301b3e93a 2024-12-11T22:59:08,691 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/a/bb4a552face942e38e982f8301b3e93a, entries=10, sequenceid=66, filesize=5.8 K 2024-12-11T22:59:08,692 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/b/e5284bda80924697b9790626b29a6101 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/b/e5284bda80924697b9790626b29a6101 2024-12-11T22:59:08,696 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/b/e5284bda80924697b9790626b29a6101, entries=10, sequenceid=66, filesize=5.8 K 2024-12-11T22:59:08,697 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/.tmp/c/30483c3e175944189acb28a3f64f03c5 as hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/c/30483c3e175944189acb28a3f64f03c5 2024-12-11T22:59:08,702 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/c/30483c3e175944189acb28a3f64f03c5, entries=10, sequenceid=66, filesize=5.8 K 2024-12-11T22:59:08,702 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3329c68d438c53ad4de408bf72dbbcdf in 95ms, sequenceid=66, compaction requested=false; wal=null 2024-12-11T22:59:08,703 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/0000000000000000066 2024-12-11T22:59:08,704 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,704 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,705 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-11T22:59:08,706 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3329c68d438c53ad4de408bf72dbbcdf 2024-12-11T22:59:08,708 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/hbase/data/default/testReplayEditsWrittenViaHRegion/3329c68d438c53ad4de408bf72dbbcdf/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-11T22:59:08,709 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3329c68d438c53ad4de408bf72dbbcdf; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71680108, jitterRate=0.06811684370040894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-11T22:59:08,709 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3329c68d438c53ad4de408bf72dbbcdf: Writing region info on filesystem at 1733957948578Initializing all the Stores at 1733957948579 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957948579Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957948579Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733957948579Obtaining lock to block concurrent updates at 1733957948607 (+28 ms)Preparing flush snapshotting stores in 3329c68d438c53ad4de408bf72dbbcdf at 1733957948607Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733957948607Flushing stores of testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. at 1733957948607Flushing 3329c68d438c53ad4de408bf72dbbcdf/a: creating writer at 1733957948607Flushing 3329c68d438c53ad4de408bf72dbbcdf/a: appending metadata at 1733957948621 (+14 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/a: closing flushed file at 1733957948621Flushing 3329c68d438c53ad4de408bf72dbbcdf/b: creating writer at 1733957948633 (+12 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/b: appending metadata at 1733957948647 (+14 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/b: closing flushed file at 1733957948647Flushing 3329c68d438c53ad4de408bf72dbbcdf/c: creating writer at 1733957948660 (+13 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/c: appending metadata at 1733957948674 (+14 ms)Flushing 3329c68d438c53ad4de408bf72dbbcdf/c: closing flushed file at 1733957948674Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b44a241: reopening flushed file at 1733957948686 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ead55b5: reopening flushed file at 1733957948691 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42b2130e: reopening flushed file at 1733957948696 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3329c68d438c53ad4de408bf72dbbcdf in 95ms, sequenceid=66, compaction requested=false; wal=null at 1733957948702 (+6 ms)Cleaning up temporary data from old regions at 1733957948704 (+2 ms)Region opened successfully at 1733957948709 (+5 ms) 2024-12-11T22:59:08,722 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3329c68d438c53ad4de408bf72dbbcdf, disabling compactions & flushes 2024-12-11T22:59:08,722 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:08,722 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:08,722 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. after waiting 0 ms 2024-12-11T22:59:08,722 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:08,724 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733957944000.3329c68d438c53ad4de408bf72dbbcdf. 2024-12-11T22:59:08,724 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3329c68d438c53ad4de408bf72dbbcdf: Waiting for close lock at 1733957948722Disabling compacts and flushes for region at 1733957948722Disabling writes for close at 1733957948722Writing region close event to WAL at 1733957948724 (+2 ms)Closed at 1733957948724 2024-12-11T22:59:08,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741939_1119 (size=93) 2024-12-11T22:59:08,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741939_1119 (size=93) 2024-12-11T22:59:08,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741939_1119 (size=93) 2024-12-11T22:59:08,738 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-11T22:59:08,738 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733957948560) 2024-12-11T22:59:08,755 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=452 (was 446) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1463521887_22 at /127.0.0.1:56098 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1463521887_22 at /127.0.0.1:37310 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1463521887_22 at /127.0.0.1:38904 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (80400340) connection to localhost/127.0.0.1:38049 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:38049 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=226 (was 228), ProcessCount=11 (was 11), AvailableMemoryMB=12044 (was 12426) 2024-12-11T22:59:08,755 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-11T22:59:08,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T22:59:08,755 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T22:59:08,755 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:59:08,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:59:08,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:59:08,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T22:59:08,756 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T22:59:08,756 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1907521982, stopped=false 2024-12-11T22:59:08,757 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b48762765ec4,38869,1733957909474 2024-12-11T22:59:08,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T22:59:08,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T22:59:08,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:59:08,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:59:08,759 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:59:08,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T22:59:08,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:59:08,759 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T22:59:08,759 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:59:08,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:59:08,760 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:59:08,760 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b48762765ec4,34571,1733957910258' ***** 2024-12-11T22:59:08,760 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T22:59:08,760 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b48762765ec4,33507,1733957910297' ***** 2024-12-11T22:59:08,760 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T22:59:08,760 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T22:59:08,761 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T22:59:08,761 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(3091): Received CLOSE for b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(959): stopping server b48762765ec4,33507,1733957910297 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(959): stopping server b48762765ec4,34571,1733957910258 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b48762765ec4:33507. 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b48762765ec4:34571. 2024-12-11T22:59:08,761 DEBUG [RS:2;b48762765ec4:33507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:59:08,761 DEBUG [RS:2;b48762765ec4:33507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:59:08,761 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b3825d650b99967f1eea9af6aad9717b, disabling compactions & flushes 2024-12-11T22:59:08,761 DEBUG [RS:1;b48762765ec4:34571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T22:59:08,761 DEBUG [RS:1;b48762765ec4:34571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:59:08,761 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T22:59:08,761 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-11T22:59:08,761 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T22:59:08,761 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1325): Online Regions={b3825d650b99967f1eea9af6aad9717b=testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b.} 2024-12-11T22:59:08,761 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. after waiting 0 ms 2024-12-11T22:59:08,761 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T22:59:08,761 DEBUG [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1351): Waiting on b3825d650b99967f1eea9af6aad9717b 2024-12-11T22:59:08,761 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:59:08,762 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T22:59:08,764 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-11T22:59:08,764 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-11T22:59:08,764 DEBUG [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T22:59:08,764 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T22:59:08,765 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T22:59:08,765 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T22:59:08,765 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T22:59:08,765 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T22:59:08,765 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-11T22:59:08,769 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/b3825d650b99967f1eea9af6aad9717b/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-11T22:59:08,770 INFO [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:59:08,770 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b3825d650b99967f1eea9af6aad9717b: Waiting for close lock at 1733957948761Running coprocessor pre-close hooks at 1733957948761Disabling compacts and flushes for region at 1733957948761Disabling writes for close at 1733957948761Writing region close event to WAL at 1733957948765 (+4 ms)Running coprocessor post-close hooks at 1733957948769 (+4 ms)Closed at 1733957948769 2024-12-11T22:59:08,770 DEBUG [RS_CLOSE_REGION-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b. 2024-12-11T22:59:08,770 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/info/9114836f7d054110bf059a747a09c6e3 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733957924387.b3825d650b99967f1eea9af6aad9717b./info:regioninfo/1733957928257/Put/seqid=0 2024-12-11T22:59:08,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741943_1123 (size=6778) 2024-12-11T22:59:08,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741943_1123 (size=6778) 2024-12-11T22:59:08,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741943_1123 (size=6778) 2024-12-11T22:59:08,777 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/info/9114836f7d054110bf059a747a09c6e3 2024-12-11T22:59:08,783 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/.tmp/info/9114836f7d054110bf059a747a09c6e3 as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/info/9114836f7d054110bf059a747a09c6e3 2024-12-11T22:59:08,788 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/info/9114836f7d054110bf059a747a09c6e3, entries=8, sequenceid=23, filesize=6.6 K 2024-12-11T22:59:08,789 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-12-11T22:59:08,789 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T22:59:08,795 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-11T22:59:08,795 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T22:59:08,796 INFO [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T22:59:08,796 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733957948764Running coprocessor pre-close hooks at 1733957948764Disabling compacts and flushes for region at 1733957948764Disabling writes for close at 1733957948765 (+1 ms)Obtaining lock to block concurrent updates at 1733957948765Preparing flush snapshotting stores in 1588230740 at 1733957948765Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733957948765Flushing stores of hbase:meta,,1.1588230740 at 1733957948766 (+1 ms)Flushing 1588230740/info: creating writer at 1733957948766Flushing 1588230740/info: appending metadata at 1733957948769 (+3 ms)Flushing 1588230740/info: closing flushed file at 1733957948769Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4efa481f: reopening flushed file at 1733957948782 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1733957948789 (+7 ms)Writing region close event to WAL at 1733957948792 (+3 ms)Running coprocessor post-close hooks at 1733957948795 (+3 ms)Closed at 1733957948795 2024-12-11T22:59:08,796 DEBUG [RS_CLOSE_META-regionserver/b48762765ec4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T22:59:08,824 INFO [regionserver/b48762765ec4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:59:08,825 INFO [regionserver/b48762765ec4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:59:08,962 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(976): stopping server b48762765ec4,34571,1733957910258; all regions closed. 2024-12-11T22:59:08,965 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(976): stopping server b48762765ec4,33507,1733957910297; all regions closed. 2024-12-11T22:59:08,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741834_1010 (size=3433) 2024-12-11T22:59:08,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741834_1010 (size=3433) 2024-12-11T22:59:08,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741834_1010 (size=3433) 2024-12-11T22:59:08,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741893_1071 (size=2422) 2024-12-11T22:59:08,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741893_1071 (size=2422) 2024-12-11T22:59:08,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741893_1071 (size=2422) 2024-12-11T22:59:08,973 DEBUG [RS:1;b48762765ec4:34571 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs 2024-12-11T22:59:08,973 INFO [RS:1;b48762765ec4:34571 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b48762765ec4%2C34571%2C1733957910258:(num 1733957911750) 2024-12-11T22:59:08,974 DEBUG [RS:1;b48762765ec4:34571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] hbase.ChoreService(370): Chore service for: regionserver/b48762765ec4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T22:59:08,974 INFO [regionserver/b48762765ec4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T22:59:08,974 INFO [RS:1;b48762765ec4:34571 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34571 2024-12-11T22:59:08,975 DEBUG [RS:2;b48762765ec4:33507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs 2024-12-11T22:59:08,975 INFO [RS:2;b48762765ec4:33507 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b48762765ec4%2C33507%2C1733957910297.meta:.meta(num 1733957927418) 2024-12-11T22:59:08,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b48762765ec4,34571,1733957910258 2024-12-11T22:59:08,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T22:59:08,977 INFO [RS:1;b48762765ec4:34571 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T22:59:08,978 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b48762765ec4,34571,1733957910258] 2024-12-11T22:59:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741833_1009 (size=93) 2024-12-11T22:59:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741833_1009 (size=93) 2024-12-11T22:59:08,979 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b48762765ec4,34571,1733957910258 already deleted, retry=false 2024-12-11T22:59:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741833_1009 (size=93) 2024-12-11T22:59:08,980 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b48762765ec4,34571,1733957910258 expired; onlineServers=1 2024-12-11T22:59:08,983 DEBUG [RS:2;b48762765ec4:33507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/oldWALs 2024-12-11T22:59:08,983 INFO [RS:2;b48762765ec4:33507 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b48762765ec4%2C33507%2C1733957910297:(num 1733957911748) 2024-12-11T22:59:08,983 DEBUG [RS:2;b48762765ec4:33507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T22:59:08,983 INFO [RS:2;b48762765ec4:33507 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T22:59:08,983 INFO [RS:2;b48762765ec4:33507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T22:59:08,983 INFO [RS:2;b48762765ec4:33507 {}] hbase.ChoreService(370): Chore service for: regionserver/b48762765ec4:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-11T22:59:08,983 INFO [RS:2;b48762765ec4:33507 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T22:59:08,983 INFO [regionserver/b48762765ec4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T22:59:08,983 INFO [RS:2;b48762765ec4:33507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33507 2024-12-11T22:59:08,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b48762765ec4,33507,1733957910297 2024-12-11T22:59:08,985 INFO [RS:2;b48762765ec4:33507 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T22:59:08,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T22:59:08,986 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b48762765ec4,33507,1733957910297] 2024-12-11T22:59:08,988 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b48762765ec4,33507,1733957910297 already deleted, retry=false 2024-12-11T22:59:08,988 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b48762765ec4,33507,1733957910297 expired; onlineServers=0 2024-12-11T22:59:08,988 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b48762765ec4,38869,1733957909474' ***** 2024-12-11T22:59:08,988 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T22:59:08,988 INFO [M:0;b48762765ec4:38869 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T22:59:08,988 INFO [M:0;b48762765ec4:38869 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T22:59:08,988 DEBUG [M:0;b48762765ec4:38869 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T22:59:08,988 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T22:59:08,989 DEBUG [M:0;b48762765ec4:38869 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T22:59:08,989 DEBUG [master/b48762765ec4:0:becomeActiveMaster-HFileCleaner.small.0-1733957911353 {}] cleaner.HFileCleaner(306): Exit Thread[master/b48762765ec4:0:becomeActiveMaster-HFileCleaner.small.0-1733957911353,5,FailOnTimeoutGroup] 2024-12-11T22:59:08,989 DEBUG [master/b48762765ec4:0:becomeActiveMaster-HFileCleaner.large.0-1733957911351 {}] cleaner.HFileCleaner(306): Exit Thread[master/b48762765ec4:0:becomeActiveMaster-HFileCleaner.large.0-1733957911351,5,FailOnTimeoutGroup] 2024-12-11T22:59:08,989 INFO [M:0;b48762765ec4:38869 {}] hbase.ChoreService(370): Chore service for: master/b48762765ec4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T22:59:08,989 INFO [M:0;b48762765ec4:38869 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T22:59:08,989 DEBUG [M:0;b48762765ec4:38869 {}] master.HMaster(1795): Stopping service threads 2024-12-11T22:59:08,989 INFO [M:0;b48762765ec4:38869 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T22:59:08,989 INFO [M:0;b48762765ec4:38869 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T22:59:08,990 INFO [M:0;b48762765ec4:38869 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T22:59:08,990 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T22:59:08,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T22:59:08,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T22:59:08,993 DEBUG [M:0;b48762765ec4:38869 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-11T22:59:08,993 DEBUG [M:0;b48762765ec4:38869 {}] master.ActiveMasterManager(353): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-11T22:59:08,994 INFO [M:0;b48762765ec4:38869 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/.lastflushedseqids 2024-12-11T22:59:09,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741944_1124 (size=119) 2024-12-11T22:59:09,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741944_1124 (size=119) 2024-12-11T22:59:09,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741944_1124 (size=119) 2024-12-11T22:59:09,015 INFO [M:0;b48762765ec4:38869 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T22:59:09,016 INFO [M:0;b48762765ec4:38869 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T22:59:09,016 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T22:59:09,016 INFO [M:0;b48762765ec4:38869 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:59:09,016 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:59:09,016 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T22:59:09,016 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:59:09,016 INFO [M:0;b48762765ec4:38869 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-12-11T22:59:09,036 DEBUG [M:0;b48762765ec4:38869 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/29e6f4615fd24000a94a329356c1f7bf is 82, key is hbase:meta,,1/info:regioninfo/1733957927596/Put/seqid=0 2024-12-11T22:59:09,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741945_1125 (size=6063) 2024-12-11T22:59:09,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741945_1125 (size=6063) 2024-12-11T22:59:09,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741945_1125 (size=6063) 2024-12-11T22:59:09,045 INFO [M:0;b48762765ec4:38869 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/29e6f4615fd24000a94a329356c1f7bf 2024-12-11T22:59:09,068 DEBUG [M:0;b48762765ec4:38869 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/368116670a8b4a9faf94da6d03f8433e is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733957924808/Put/seqid=0 2024-12-11T22:59:09,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:59:09,079 INFO [RS:1;b48762765ec4:34571 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T22:59:09,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34571-0x100cb900b300002, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:59:09,079 INFO [RS:1;b48762765ec4:34571 {}] regionserver.HRegionServer(1031): Exiting; stopping=b48762765ec4,34571,1733957910258; zookeeper connection closed. 2024-12-11T22:59:09,084 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58dd5213 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58dd5213 2024-12-11T22:59:09,087 INFO [RS:2;b48762765ec4:33507 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T22:59:09,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:59:09,087 INFO [RS:2;b48762765ec4:33507 {}] regionserver.HRegionServer(1031): Exiting; stopping=b48762765ec4,33507,1733957910297; zookeeper connection closed. 2024-12-11T22:59:09,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33507-0x100cb900b300003, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:59:09,088 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73c90962 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73c90962 2024-12-11T22:59:09,089 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-11T22:59:09,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741946_1126 (size=7907) 2024-12-11T22:59:09,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741946_1126 (size=7907) 2024-12-11T22:59:09,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741946_1126 (size=7907) 2024-12-11T22:59:09,091 INFO [M:0;b48762765ec4:38869 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/368116670a8b4a9faf94da6d03f8433e 2024-12-11T22:59:09,097 INFO [M:0;b48762765ec4:38869 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 368116670a8b4a9faf94da6d03f8433e 2024-12-11T22:59:09,115 DEBUG [M:0;b48762765ec4:38869 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3c52dd075bfa41dba5649e0f7789410b is 69, key is b48762765ec4,33507,1733957910297/rs:state/1733957911495/Put/seqid=0 2024-12-11T22:59:09,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741947_1127 (size=5440) 2024-12-11T22:59:09,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741947_1127 (size=5440) 2024-12-11T22:59:09,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741947_1127 (size=5440) 2024-12-11T22:59:09,123 INFO [M:0;b48762765ec4:38869 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3c52dd075bfa41dba5649e0f7789410b 2024-12-11T22:59:09,129 INFO [M:0;b48762765ec4:38869 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3c52dd075bfa41dba5649e0f7789410b 2024-12-11T22:59:09,130 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/29e6f4615fd24000a94a329356c1f7bf as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/29e6f4615fd24000a94a329356c1f7bf 2024-12-11T22:59:09,135 INFO [M:0;b48762765ec4:38869 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/29e6f4615fd24000a94a329356c1f7bf, entries=14, sequenceid=207, filesize=5.9 K 2024-12-11T22:59:09,136 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/368116670a8b4a9faf94da6d03f8433e as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/368116670a8b4a9faf94da6d03f8433e 2024-12-11T22:59:09,142 INFO [M:0;b48762765ec4:38869 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 368116670a8b4a9faf94da6d03f8433e 2024-12-11T22:59:09,142 INFO [M:0;b48762765ec4:38869 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/368116670a8b4a9faf94da6d03f8433e, entries=21, sequenceid=207, filesize=7.7 K 2024-12-11T22:59:09,143 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3c52dd075bfa41dba5649e0f7789410b as hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3c52dd075bfa41dba5649e0f7789410b 2024-12-11T22:59:09,167 INFO [M:0;b48762765ec4:38869 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3c52dd075bfa41dba5649e0f7789410b 2024-12-11T22:59:09,167 INFO [M:0;b48762765ec4:38869 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38049/user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3c52dd075bfa41dba5649e0f7789410b, entries=3, sequenceid=207, filesize=5.3 K 2024-12-11T22:59:09,170 INFO [M:0;b48762765ec4:38869 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 154ms, sequenceid=207, compaction requested=false 2024-12-11T22:59:09,177 INFO [M:0;b48762765ec4:38869 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T22:59:09,177 DEBUG [M:0;b48762765ec4:38869 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733957949016Disabling compacts and flushes for region at 1733957949016Disabling writes for close at 1733957949016Obtaining lock to block concurrent updates at 1733957949016Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733957949016Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1733957949016Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733957949017 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733957949017Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733957949036 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733957949036Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733957949051 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733957949067 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733957949067Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733957949097 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733957949114 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733957949114Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e1252a3: reopening flushed file at 1733957949129 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c2cb4cd: reopening flushed file at 1733957949136 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ac3fdd0: reopening flushed file at 1733957949142 (+6 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 154ms, sequenceid=207, compaction requested=false at 1733957949170 (+28 ms)Writing region close event to WAL at 1733957949176 (+6 ms)Closed at 1733957949176 2024-12-11T22:59:09,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43249 is added to blk_1073741830_1006 (size=99436) 2024-12-11T22:59:09,220 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/68e943b7-6d9e-cc91-29c6-df3f4612255c/MasterData/WALs/b48762765ec4,38869,1733957909474/b48762765ec4%2C38869%2C1733957909474.1733957910733 not finished, retry = 0 2024-12-11T22:59:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37177 is added to blk_1073741830_1006 (size=99436) 2024-12-11T22:59:09,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39291 is added to blk_1073741830_1006 (size=99436) 2024-12-11T22:59:09,322 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T22:59:09,322 INFO [M:0;b48762765ec4:38869 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-11T22:59:09,322 INFO [M:0;b48762765ec4:38869 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38869 2024-12-11T22:59:09,322 INFO [M:0;b48762765ec4:38869 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T22:59:09,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:59:09,424 INFO [M:0;b48762765ec4:38869 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T22:59:09,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38869-0x100cb900b300000, quorum=127.0.0.1:52314, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T22:59:09,432 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957938619 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957938619 (inode 16688) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733957928934/wal.1733957938619 (inode 16688) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-11T22:59:09,435 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928768 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928768 (inode 16665) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733957928476/wal.1733957928768 (inode 16665) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-11T22:59:09,436 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733957919868/wal.1733957919974 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:09,439 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939439 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939439 (inode 16714) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733957938898/wal.1733957939439 (inode 16714) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-11T22:59:09,440 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733957939580/wal.1733957939633 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:09,440 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733957943998/wal.1733957944276 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:09,440 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733957913337/wal.1733957913411 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T22:59:09,443 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal.1733957913208 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal.1733957913208 (inode 16485) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733957913024/wal.1733957913208 (inode 16485) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-11T22:59:09,446 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal.1733957912774 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal.1733957912774 (inode 16462) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733957912527/wal.1733957912774 (inode 16462) Holder DFSClient_NONMAPREDUCE_866042119_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-11T22:59:09,458 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3e2333e3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:59:09,461 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b860b3d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T22:59:09,461 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T22:59:09,461 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e5ef071{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T22:59:09,462 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4752d0b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,STOPPED} 2024-12-11T22:59:09,467 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-11T22:59:09,467 WARN [BP-1204600519-172.17.0.3-1733957906467 heartbeating to localhost/127.0.0.1:38049 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-11T22:59:09,467 WARN [BP-1204600519-172.17.0.3-1733957906467 heartbeating to localhost/127.0.0.1:38049 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1204600519-172.17.0.3-1733957906467 (Datanode Uuid 422e996f-6fdd-435a-bd71-360e657922dd) service to localhost/127.0.0.1:38049 2024-12-11T22:59:09,467 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-11T22:59:09,468 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data5/current/BP-1204600519-172.17.0.3-1733957906467 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T22:59:09,469 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data6/current/BP-1204600519-172.17.0.3-1733957906467 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T22:59:09,469 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-11T22:59:09,473 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@36c913bb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:59:09,473 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a0e4ecc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T22:59:09,474 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T22:59:09,474 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@77f84511{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T22:59:09,474 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1919f676{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,STOPPED} 2024-12-11T22:59:09,476 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-11T22:59:09,476 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-11T22:59:09,476 WARN [BP-1204600519-172.17.0.3-1733957906467 heartbeating to localhost/127.0.0.1:38049 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-11T22:59:09,476 WARN [BP-1204600519-172.17.0.3-1733957906467 heartbeating to localhost/127.0.0.1:38049 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1204600519-172.17.0.3-1733957906467 (Datanode Uuid 0d606b3a-f8db-4a97-a721-5950cf4fad3a) service to localhost/127.0.0.1:38049 2024-12-11T22:59:09,477 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data3/current/BP-1204600519-172.17.0.3-1733957906467 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T22:59:09,477 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data4/current/BP-1204600519-172.17.0.3-1733957906467 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T22:59:09,479 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-11T22:59:09,482 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c65e35b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T22:59:09,483 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@67cd2c22{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T22:59:09,483 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T22:59:09,483 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74576bd2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T22:59:09,483 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@734853a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,STOPPED} 2024-12-11T22:59:09,485 WARN [BP-1204600519-172.17.0.3-1733957906467 heartbeating to localhost/127.0.0.1:38049 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-11T22:59:09,485 WARN [BP-1204600519-172.17.0.3-1733957906467 heartbeating to localhost/127.0.0.1:38049 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1204600519-172.17.0.3-1733957906467 (Datanode Uuid 71b54c86-2458-4d06-9d91-d41840b55c89) service to localhost/127.0.0.1:38049 2024-12-11T22:59:09,486 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data2/current/BP-1204600519-172.17.0.3-1733957906467 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T22:59:09,486 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-11T22:59:09,486 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-11T22:59:09,486 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/cluster_ec04c958-5fab-f637-d730-11ddc16eb028/data/data1/current/BP-1204600519-172.17.0.3-1733957906467 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T22:59:09,486 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-11T22:59:09,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@25dc94e0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T22:59:09,497 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7cc108fc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T22:59:09,497 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T22:59:09,497 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@377439aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T22:59:09,497 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39b35319{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1003dc49-a094-7754-bdef-563a84fc6d53/hadoop.log.dir/,STOPPED} 2024-12-11T22:59:09,509 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-11T22:59:09,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down