2024-12-05 13:27:31,487 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-05 13:27:31,497 main DEBUG Took 0.008234 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 13:27:31,498 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 13:27:31,498 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 13:27:31,499 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 13:27:31,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,505 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 13:27:31,517 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,519 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,520 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,521 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,521 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,522 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,523 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,523 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,524 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,524 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,525 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:27:31,525 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,526 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 13:27:31,527 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:27:31,528 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 13:27:31,530 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 13:27:31,530 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 13:27:31,531 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 13:27:31,532 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 13:27:31,539 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 13:27:31,541 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 13:27:31,543 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 13:27:31,543 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 13:27:31,543 main DEBUG createAppenders(={Console}) 2024-12-05 13:27:31,544 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-12-05 13:27:31,544 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-12-05 13:27:31,544 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-12-05 13:27:31,545 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 13:27:31,545 main DEBUG OutputStream closed 2024-12-05 13:27:31,545 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 13:27:31,545 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 13:27:31,545 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-12-05 13:27:31,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 13:27:31,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 13:27:31,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 13:27:31,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 13:27:31,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 13:27:31,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 13:27:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 13:27:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 13:27:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 13:27:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 13:27:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 13:27:31,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 13:27:31,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 13:27:31,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 13:27:31,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 13:27:31,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 13:27:31,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 13:27:31,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 13:27:31,621 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 13:27:31,621 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-12-05 13:27:31,621 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 13:27:31,622 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-12-05T13:27:31,820 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a 2024-12-05 13:27:31,823 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 13:27:31,823 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T13:27:31,832 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-05T13:27:31,861 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T13:27:31,908 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T13:27:31,909 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T13:27:31,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T13:27:31,931 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d, deleteOnExit=true 2024-12-05T13:27:31,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T13:27:31,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/test.cache.data in system properties and HBase conf 2024-12-05T13:27:31,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T13:27:31,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir in system properties and HBase conf 2024-12-05T13:27:31,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T13:27:31,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T13:27:31,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T13:27:32,008 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T13:27:32,104 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T13:27:32,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T13:27:32,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T13:27:32,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T13:27:32,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T13:27:32,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T13:27:32,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T13:27:32,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T13:27:32,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T13:27:32,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T13:27:32,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/nfs.dump.dir in system properties and HBase conf 2024-12-05T13:27:32,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/java.io.tmpdir in system properties and HBase conf 2024-12-05T13:27:32,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T13:27:32,111 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T13:27:32,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T13:27:32,990 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T13:27:33,071 INFO [Time-limited test {}] log.Log(170): Logging initialized @2199ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T13:27:33,147 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:27:33,211 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:27:33,239 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:27:33,240 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:27:33,241 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T13:27:33,252 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:27:33,254 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:27:33,255 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:27:33,400 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/java.io.tmpdir/jetty-localhost-39999-hadoop-hdfs-3_4_1-tests_jar-_-any-7511238012793224426/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T13:27:33,405 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:39999} 2024-12-05T13:27:33,406 INFO [Time-limited test {}] server.Server(415): Started @2533ms 2024-12-05T13:27:33,954 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:27:33,963 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:27:33,964 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:27:33,964 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:27:33,964 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T13:27:33,965 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29f9ab09{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:27:33,966 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c19a0d8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:27:34,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5bb7bef3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/java.io.tmpdir/jetty-localhost-36111-hadoop-hdfs-3_4_1-tests_jar-_-any-6193400594597326364/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:27:34,074 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4080a860{HTTP/1.1, (http/1.1)}{localhost:36111} 2024-12-05T13:27:34,074 INFO [Time-limited test {}] server.Server(415): Started @3202ms 2024-12-05T13:27:34,140 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T13:27:34,300 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:27:34,307 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:27:34,311 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:27:34,312 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:27:34,312 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T13:27:34,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fc499ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:27:34,314 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@152e2397{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:27:34,443 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@494486cb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/java.io.tmpdir/jetty-localhost-46205-hadoop-hdfs-3_4_1-tests_jar-_-any-7917590808995258726/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:27:34,444 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d076b20{HTTP/1.1, (http/1.1)}{localhost:46205} 2024-12-05T13:27:34,444 INFO [Time-limited test {}] server.Server(415): Started @3572ms 2024-12-05T13:27:34,447 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T13:27:34,485 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:27:34,490 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:27:34,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:27:34,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:27:34,492 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T13:27:34,493 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37d4eea4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:27:34,493 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f8696d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:27:34,586 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50b13206{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/java.io.tmpdir/jetty-localhost-34711-hadoop-hdfs-3_4_1-tests_jar-_-any-15134210733186487957/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:27:34,587 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3267e685{HTTP/1.1, (http/1.1)}{localhost:34711} 2024-12-05T13:27:34,588 INFO [Time-limited test {}] server.Server(415): Started @3715ms 2024-12-05T13:27:34,590 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T13:27:35,373 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data4/current/BP-1492241733-172.17.0.3-1733405252624/current, will proceed with Du for space computation calculation, 2024-12-05T13:27:35,373 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data2/current/BP-1492241733-172.17.0.3-1733405252624/current, will proceed with Du for space computation calculation, 2024-12-05T13:27:35,373 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data3/current/BP-1492241733-172.17.0.3-1733405252624/current, will proceed with Du for space computation calculation, 2024-12-05T13:27:35,373 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data1/current/BP-1492241733-172.17.0.3-1733405252624/current, will proceed with Du for space computation calculation, 2024-12-05T13:27:35,402 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T13:27:35,402 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T13:27:35,447 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4bda1cfa9ee1bc51 with lease ID 0xa2622398b6909340: Processing first storage report for DS-5fe20c54-5be8-4e68-a457-25143a1f9317 from datanode DatanodeRegistration(127.0.0.1:40037, datanodeUuid=392a5c38-1118-42f2-a5b7-5c852f57abb6, infoPort=32953, infoSecurePort=0, ipcPort=42385, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624) 2024-12-05T13:27:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4bda1cfa9ee1bc51 with lease ID 0xa2622398b6909340: from storage DS-5fe20c54-5be8-4e68-a457-25143a1f9317 node DatanodeRegistration(127.0.0.1:40037, datanodeUuid=392a5c38-1118-42f2-a5b7-5c852f57abb6, infoPort=32953, infoSecurePort=0, ipcPort=42385, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T13:27:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ea4fb74d63d3462 with lease ID 0xa2622398b6909341: Processing first storage report for DS-153a313f-f6ef-473d-84f3-deb45a40d480 from datanode DatanodeRegistration(127.0.0.1:38677, datanodeUuid=26f974ac-76cd-4a28-932e-c73f194b6651, infoPort=35457, infoSecurePort=0, ipcPort=36661, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624) 2024-12-05T13:27:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ea4fb74d63d3462 with lease ID 0xa2622398b6909341: from storage DS-153a313f-f6ef-473d-84f3-deb45a40d480 node DatanodeRegistration(127.0.0.1:38677, datanodeUuid=26f974ac-76cd-4a28-932e-c73f194b6651, infoPort=35457, infoSecurePort=0, ipcPort=36661, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T13:27:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4bda1cfa9ee1bc51 with lease ID 0xa2622398b6909340: Processing first storage report for DS-00289e95-611a-4853-a65a-2d11c69e973d from datanode DatanodeRegistration(127.0.0.1:40037, datanodeUuid=392a5c38-1118-42f2-a5b7-5c852f57abb6, infoPort=32953, infoSecurePort=0, ipcPort=42385, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624) 2024-12-05T13:27:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4bda1cfa9ee1bc51 with lease ID 0xa2622398b6909340: from storage DS-00289e95-611a-4853-a65a-2d11c69e973d node DatanodeRegistration(127.0.0.1:40037, datanodeUuid=392a5c38-1118-42f2-a5b7-5c852f57abb6, infoPort=32953, infoSecurePort=0, ipcPort=42385, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T13:27:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ea4fb74d63d3462 with lease ID 0xa2622398b6909341: Processing first storage report for DS-c450e310-8093-4ebb-a866-5f0e636b5ead from datanode DatanodeRegistration(127.0.0.1:38677, datanodeUuid=26f974ac-76cd-4a28-932e-c73f194b6651, infoPort=35457, infoSecurePort=0, ipcPort=36661, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624) 2024-12-05T13:27:35,450 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ea4fb74d63d3462 with lease ID 0xa2622398b6909341: from storage DS-c450e310-8093-4ebb-a866-5f0e636b5ead node DatanodeRegistration(127.0.0.1:38677, datanodeUuid=26f974ac-76cd-4a28-932e-c73f194b6651, infoPort=35457, infoSecurePort=0, ipcPort=36661, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T13:27:35,450 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data5/current/BP-1492241733-172.17.0.3-1733405252624/current, will proceed with Du for space computation calculation, 2024-12-05T13:27:35,451 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data6/current/BP-1492241733-172.17.0.3-1733405252624/current, will proceed with Du for space computation calculation, 2024-12-05T13:27:35,471 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T13:27:35,477 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2da8552b766bab2a with lease ID 0xa2622398b6909342: Processing first storage report for DS-5be99c92-5113-4350-9597-c1ed3fbf9853 from datanode DatanodeRegistration(127.0.0.1:43089, datanodeUuid=8fff9606-6a5a-4fcd-ae2e-b4b7365386ef, infoPort=38653, infoSecurePort=0, ipcPort=36613, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624) 2024-12-05T13:27:35,478 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2da8552b766bab2a with lease ID 0xa2622398b6909342: from storage DS-5be99c92-5113-4350-9597-c1ed3fbf9853 node DatanodeRegistration(127.0.0.1:43089, datanodeUuid=8fff9606-6a5a-4fcd-ae2e-b4b7365386ef, infoPort=38653, infoSecurePort=0, ipcPort=36613, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T13:27:35,478 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2da8552b766bab2a with lease ID 0xa2622398b6909342: Processing first storage report for DS-3a39e4b4-a8e9-44fd-a405-ae8fa2b682b5 from datanode DatanodeRegistration(127.0.0.1:43089, datanodeUuid=8fff9606-6a5a-4fcd-ae2e-b4b7365386ef, infoPort=38653, infoSecurePort=0, ipcPort=36613, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624) 2024-12-05T13:27:35,478 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2da8552b766bab2a with lease ID 0xa2622398b6909342: from storage DS-3a39e4b4-a8e9-44fd-a405-ae8fa2b682b5 node DatanodeRegistration(127.0.0.1:43089, datanodeUuid=8fff9606-6a5a-4fcd-ae2e-b4b7365386ef, infoPort=38653, infoSecurePort=0, ipcPort=36613, storageInfo=lv=-57;cid=testClusterID;nsid=1505476733;c=1733405252624), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T13:27:35,514 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a 2024-12-05T13:27:35,599 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/zookeeper_0, clientPort=61512, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T13:27:35,608 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61512 2024-12-05T13:27:35,618 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:35,621 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:35,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741825_1001 (size=7) 2024-12-05T13:27:35,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741825_1001 (size=7) 2024-12-05T13:27:35,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741825_1001 (size=7) 2024-12-05T13:27:36,218 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c with version=8 2024-12-05T13:27:36,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/hbase-staging 2024-12-05T13:27:36,480 INFO [Time-limited test {}] client.ConnectionUtils(128): master/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:27:36,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:36,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:36,494 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:27:36,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:36,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:27:36,616 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T13:27:36,673 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T13:27:36,682 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T13:27:36,686 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:27:36,713 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 48716 (auto-detected) 2024-12-05T13:27:36,714 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-05T13:27:36,732 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43665 2024-12-05T13:27:36,749 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43665 connecting to ZooKeeper ensemble=127.0.0.1:61512 2024-12-05T13:27:36,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436650x0, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:27:36,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43665-0x101a6daed470000 connected 2024-12-05T13:27:37,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,066 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,076 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:27:37,080 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c, hbase.cluster.distributed=false 2024-12-05T13:27:37,102 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:27:37,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43665 2024-12-05T13:27:37,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43665 2024-12-05T13:27:37,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43665 2024-12-05T13:27:37,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43665 2024-12-05T13:27:37,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43665 2024-12-05T13:27:37,200 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:27:37,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,202 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:27:37,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:27:37,205 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T13:27:37,207 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:27:37,208 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33089 2024-12-05T13:27:37,209 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33089 connecting to ZooKeeper ensemble=127.0.0.1:61512 2024-12-05T13:27:37,211 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,213 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330890x0, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:27:37,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33089-0x101a6daed470001 connected 2024-12-05T13:27:37,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:27:37,265 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T13:27:37,273 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T13:27:37,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T13:27:37,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:27:37,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33089 2024-12-05T13:27:37,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33089 2024-12-05T13:27:37,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33089 2024-12-05T13:27:37,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33089 2024-12-05T13:27:37,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33089 2024-12-05T13:27:37,297 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:27:37,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,298 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:27:37,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,298 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:27:37,299 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T13:27:37,299 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:27:37,300 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35441 2024-12-05T13:27:37,301 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35441 connecting to ZooKeeper ensemble=127.0.0.1:61512 2024-12-05T13:27:37,302 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,304 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354410x0, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:27:37,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35441-0x101a6daed470002 connected 2024-12-05T13:27:37,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:27:37,323 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T13:27:37,324 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T13:27:37,325 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T13:27:37,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:27:37,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35441 2024-12-05T13:27:37,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35441 2024-12-05T13:27:37,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35441 2024-12-05T13:27:37,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35441 2024-12-05T13:27:37,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35441 2024-12-05T13:27:37,343 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:27:37,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,344 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:27:37,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:27:37,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:27:37,344 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T13:27:37,344 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:27:37,345 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44873 2024-12-05T13:27:37,346 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44873 connecting to ZooKeeper ensemble=127.0.0.1:61512 2024-12-05T13:27:37,347 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,349 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448730x0, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:27:37,360 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:448730x0, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:27:37,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44873-0x101a6daed470003 connected 2024-12-05T13:27:37,361 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T13:27:37,361 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T13:27:37,362 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T13:27:37,364 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:27:37,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44873 2024-12-05T13:27:37,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44873 2024-12-05T13:27:37,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44873 2024-12-05T13:27:37,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44873 2024-12-05T13:27:37,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44873 2024-12-05T13:27:37,381 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;88aaa70743e7:43665 2024-12-05T13:27:37,382 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/88aaa70743e7,43665,1733405256332 2024-12-05T13:27:37,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,399 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/88aaa70743e7,43665,1733405256332 2024-12-05T13:27:37,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T13:27:37,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T13:27:37,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T13:27:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,431 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T13:27:37,432 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/88aaa70743e7,43665,1733405256332 from backup master directory 2024-12-05T13:27:37,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/88aaa70743e7,43665,1733405256332 2024-12-05T13:27:37,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:27:37,444 WARN [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:27:37,444 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=88aaa70743e7,43665,1733405256332 2024-12-05T13:27:37,447 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T13:27:37,448 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T13:27:37,501 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/hbase.id] with ID: 3e1a98ee-101f-4ab3-b69a-238955073761 2024-12-05T13:27:37,501 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/.tmp/hbase.id 2024-12-05T13:27:37,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741826_1002 (size=42) 2024-12-05T13:27:37,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741826_1002 (size=42) 2024-12-05T13:27:37,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741826_1002 (size=42) 2024-12-05T13:27:37,515 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/.tmp/hbase.id]:[hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/hbase.id] 2024-12-05T13:27:37,560 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:27:37,564 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T13:27:37,581 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-05T13:27:37,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:37,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741827_1003 (size=196) 2024-12-05T13:27:37,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741827_1003 (size=196) 2024-12-05T13:27:37,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741827_1003 (size=196) 2024-12-05T13:27:37,625 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T13:27:37,627 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T13:27:37,633 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:37,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741828_1004 (size=1189) 2024-12-05T13:27:37,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741828_1004 (size=1189) 2024-12-05T13:27:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741828_1004 (size=1189) 2024-12-05T13:27:37,682 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store 2024-12-05T13:27:37,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741829_1005 (size=34) 2024-12-05T13:27:37,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741829_1005 (size=34) 2024-12-05T13:27:37,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741829_1005 (size=34) 2024-12-05T13:27:37,712 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T13:27:37,717 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:37,718 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T13:27:37,719 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:27:37,719 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:27:37,721 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T13:27:37,722 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:27:37,722 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:27:37,724 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733405257718Disabling compacts and flushes for region at 1733405257718Disabling writes for close at 1733405257722 (+4 ms)Writing region close event to WAL at 1733405257722Closed at 1733405257722 2024-12-05T13:27:37,727 WARN [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/.initializing 2024-12-05T13:27:37,727 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/WALs/88aaa70743e7,43665,1733405256332 2024-12-05T13:27:37,737 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:37,752 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C43665%2C1733405256332, suffix=, logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/WALs/88aaa70743e7,43665,1733405256332, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/oldWALs, maxLogs=10 2024-12-05T13:27:37,786 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/WALs/88aaa70743e7,43665,1733405256332/88aaa70743e7%2C43665%2C1733405256332.1733405257757, exclude list is [], retry=0 2024-12-05T13:27:37,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:37,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:37,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:37,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:37,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T13:27:37,849 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/WALs/88aaa70743e7,43665,1733405256332/88aaa70743e7%2C43665%2C1733405256332.1733405257757 2024-12-05T13:27:37,850 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:27:37,850 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:37,850 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:37,853 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,854 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,910 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T13:27:37,914 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:37,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:37,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T13:27:37,920 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:37,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:37,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T13:27:37,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:37,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:37,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T13:27:37,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:37,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:37,929 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,933 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,934 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,940 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,940 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,945 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:27:37,950 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:27:37,954 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:37,956 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68460837, jitterRate=0.020145967602729797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:27:37,961 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733405257865Initializing all the Stores at 1733405257867 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405257868 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405257868Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405257869 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405257869Cleaning up temporary data from old regions at 1733405257941 (+72 ms)Region opened successfully at 1733405257961 (+20 ms) 2024-12-05T13:27:37,963 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T13:27:37,996 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b93616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:27:38,025 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T13:27:38,034 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T13:27:38,034 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T13:27:38,037 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:27:38,038 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:27:38,043 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T13:27:38,043 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T13:27:38,070 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T13:27:38,078 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T13:27:38,112 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T13:27:38,116 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T13:27:38,119 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T13:27:38,129 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T13:27:38,133 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T13:27:38,136 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T13:27:38,146 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T13:27:38,147 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T13:27:38,154 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T13:27:38,176 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T13:27:38,184 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T13:27:38,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:27:38,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:27:38,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:27:38,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:27:38,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,201 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=88aaa70743e7,43665,1733405256332, sessionid=0x101a6daed470000, setting cluster-up flag (Was=false) 2024-12-05T13:27:38,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,254 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T13:27:38,257 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88aaa70743e7,43665,1733405256332 2024-12-05T13:27:38,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:38,304 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T13:27:38,306 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88aaa70743e7,43665,1733405256332 2024-12-05T13:27:38,312 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T13:27:38,372 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(746): ClusterId : 3e1a98ee-101f-4ab3-b69a-238955073761 2024-12-05T13:27:38,372 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(746): ClusterId : 3e1a98ee-101f-4ab3-b69a-238955073761 2024-12-05T13:27:38,372 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(746): ClusterId : 3e1a98ee-101f-4ab3-b69a-238955073761 2024-12-05T13:27:38,375 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T13:27:38,375 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T13:27:38,375 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T13:27:38,392 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:27:38,395 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T13:27:38,395 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T13:27:38,395 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T13:27:38,395 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T13:27:38,395 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T13:27:38,395 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T13:27:38,402 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T13:27:38,412 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T13:27:38,414 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T13:27:38,415 DEBUG [RS:1;88aaa70743e7:35441 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc3515c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:27:38,416 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T13:27:38,416 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T13:27:38,417 DEBUG [RS:2;88aaa70743e7:44873 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f1ada9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:27:38,417 DEBUG [RS:0;88aaa70743e7:33089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3604baa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:27:38,420 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 88aaa70743e7,43665,1733405256332 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T13:27:38,430 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:27:38,430 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:27:38,430 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:27:38,430 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:27:38,431 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/88aaa70743e7:0, corePoolSize=10, maxPoolSize=10 2024-12-05T13:27:38,431 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,431 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:27:38,431 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,433 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;88aaa70743e7:35441 2024-12-05T13:27:38,437 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T13:27:38,438 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T13:27:38,438 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T13:27:38,443 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,43665,1733405256332 with port=35441, startcode=1733405257297 2024-12-05T13:27:38,444 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;88aaa70743e7:44873 2024-12-05T13:27:38,444 DEBUG [RS:0;88aaa70743e7:33089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;88aaa70743e7:33089 2024-12-05T13:27:38,444 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T13:27:38,444 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T13:27:38,444 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T13:27:38,444 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T13:27:38,444 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T13:27:38,444 DEBUG [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T13:27:38,445 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,43665,1733405256332 with port=33089, startcode=1733405257168 2024-12-05T13:27:38,445 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,43665,1733405256332 with port=44873, startcode=1733405257343 2024-12-05T13:27:38,457 DEBUG [RS:0;88aaa70743e7:33089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T13:27:38,457 DEBUG [RS:2;88aaa70743e7:44873 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T13:27:38,457 DEBUG [RS:1;88aaa70743e7:35441 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T13:27:38,463 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T13:27:38,464 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T13:27:38,472 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:38,472 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T13:27:38,475 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733405288475 2024-12-05T13:27:38,477 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T13:27:38,478 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T13:27:38,483 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T13:27:38,484 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T13:27:38,484 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T13:27:38,485 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T13:27:38,491 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,499 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T13:27:38,524 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T13:27:38,524 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T13:27:38,527 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T13:27:38,528 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T13:27:38,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741831_1007 (size=1321) 2024-12-05T13:27:38,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741831_1007 (size=1321) 2024-12-05T13:27:38,531 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.large.0-1733405258531,5,FailOnTimeoutGroup] 2024-12-05T13:27:38,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741831_1007 (size=1321) 2024-12-05T13:27:38,533 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.small.0-1733405258532,5,FailOnTimeoutGroup] 2024-12-05T13:27:38,533 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,533 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T13:27:38,533 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T13:27:38,533 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46783, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T13:27:38,534 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c 2024-12-05T13:27:38,535 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,535 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56651, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T13:27:38,535 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39481, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T13:27:38,535 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,542 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43665 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88aaa70743e7,44873,1733405257343 2024-12-05T13:27:38,545 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43665 {}] master.ServerManager(517): Registering regionserver=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:38,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741832_1008 (size=32) 2024-12-05T13:27:38,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741832_1008 (size=32) 2024-12-05T13:27:38,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741832_1008 (size=32) 2024-12-05T13:27:38,558 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:38,561 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43665 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88aaa70743e7,35441,1733405257297 2024-12-05T13:27:38,561 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43665 {}] master.ServerManager(517): Registering regionserver=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:38,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T13:27:38,565 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T13:27:38,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:38,566 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c 2024-12-05T13:27:38,566 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44203 2024-12-05T13:27:38,567 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T13:27:38,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:38,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T13:27:38,568 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43665 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88aaa70743e7,33089,1733405257168 2024-12-05T13:27:38,568 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43665 {}] master.ServerManager(517): Registering regionserver=88aaa70743e7,33089,1733405257168 2024-12-05T13:27:38,569 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c 2024-12-05T13:27:38,569 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44203 2024-12-05T13:27:38,569 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T13:27:38,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T13:27:38,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:38,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:38,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T13:27:38,573 DEBUG [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c 2024-12-05T13:27:38,573 DEBUG [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44203 2024-12-05T13:27:38,573 DEBUG [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T13:27:38,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T13:27:38,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:38,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:38,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T13:27:38,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T13:27:38,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T13:27:38,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:38,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:38,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T13:27:38,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740 2024-12-05T13:27:38,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740 2024-12-05T13:27:38,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T13:27:38,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T13:27:38,590 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:27:38,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T13:27:38,598 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:38,599 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74862704, jitterRate=0.11554121971130371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:27:38,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733405258558Initializing all the Stores at 1733405258561 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405258561Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405258561Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405258561Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405258561Cleaning up temporary data from old regions at 1733405258589 (+28 ms)Region opened successfully at 1733405258601 (+12 ms) 2024-12-05T13:27:38,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T13:27:38,601 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T13:27:38,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T13:27:38,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T13:27:38,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T13:27:38,603 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T13:27:38,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733405258601Disabling compacts and flushes for region at 1733405258601Disabling writes for close at 1733405258602 (+1 ms)Writing region close event to WAL at 1733405258603 (+1 ms)Closed at 1733405258603 2024-12-05T13:27:38,606 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T13:27:38,606 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T13:27:38,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T13:27:38,621 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:27:38,624 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T13:27:38,625 DEBUG [RS:2;88aaa70743e7:44873 {}] zookeeper.ZKUtil(111): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88aaa70743e7,44873,1733405257343 2024-12-05T13:27:38,626 WARN [RS:2;88aaa70743e7:44873 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:27:38,626 INFO [RS:2;88aaa70743e7:44873 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:38,626 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,44873,1733405257343 2024-12-05T13:27:38,626 DEBUG [RS:1;88aaa70743e7:35441 {}] zookeeper.ZKUtil(111): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88aaa70743e7,35441,1733405257297 2024-12-05T13:27:38,626 WARN [RS:1;88aaa70743e7:35441 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:27:38,627 INFO [RS:1;88aaa70743e7:35441 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:38,627 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297 2024-12-05T13:27:38,628 DEBUG [RS:0;88aaa70743e7:33089 {}] zookeeper.ZKUtil(111): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88aaa70743e7,33089,1733405257168 2024-12-05T13:27:38,628 WARN [RS:0;88aaa70743e7:33089 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:27:38,628 INFO [RS:0;88aaa70743e7:33089 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:38,629 DEBUG [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,33089,1733405257168 2024-12-05T13:27:38,636 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88aaa70743e7,33089,1733405257168] 2024-12-05T13:27:38,636 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88aaa70743e7,44873,1733405257343] 2024-12-05T13:27:38,636 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88aaa70743e7,35441,1733405257297] 2024-12-05T13:27:38,661 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T13:27:38,661 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T13:27:38,661 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T13:27:38,677 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T13:27:38,677 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T13:27:38,677 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T13:27:38,691 INFO [RS:0;88aaa70743e7:33089 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:27:38,691 INFO [RS:2;88aaa70743e7:44873 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:27:38,691 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,691 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,695 INFO [RS:1;88aaa70743e7:35441 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:27:38,695 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,696 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T13:27:38,696 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T13:27:38,696 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T13:27:38,702 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T13:27:38,702 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T13:27:38,702 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T13:27:38,703 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,703 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,703 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,704 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,704 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:27:38,704 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:27:38,704 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:27:38,704 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:2;88aaa70743e7:44873 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:27:38,705 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:27:38,705 DEBUG [RS:0;88aaa70743e7:33089 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:27:38,706 DEBUG [RS:1;88aaa70743e7:35441 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:27:38,709 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,33089,1733405257168-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,709 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,710 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,35441,1733405257297-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:27:38,710 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,44873,1733405257343-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:27:38,730 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T13:27:38,731 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T13:27:38,731 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T13:27:38,732 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,44873,1733405257343-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,732 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,35441,1733405257297-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,732 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,33089,1733405257168-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,733 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,733 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,733 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,733 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.Replication(171): 88aaa70743e7,35441,1733405257297 started 2024-12-05T13:27:38,733 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.Replication(171): 88aaa70743e7,44873,1733405257343 started 2024-12-05T13:27:38,733 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.Replication(171): 88aaa70743e7,33089,1733405257168 started 2024-12-05T13:27:38,749 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,750 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1482): Serving as 88aaa70743e7,44873,1733405257343, RpcServer on 88aaa70743e7/172.17.0.3:44873, sessionid=0x101a6daed470003 2024-12-05T13:27:38,750 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T13:27:38,750 DEBUG [RS:2;88aaa70743e7:44873 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88aaa70743e7,44873,1733405257343 2024-12-05T13:27:38,751 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,44873,1733405257343' 2024-12-05T13:27:38,751 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T13:27:38,752 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T13:27:38,752 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T13:27:38,752 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T13:27:38,752 DEBUG [RS:2;88aaa70743e7:44873 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88aaa70743e7,44873,1733405257343 2024-12-05T13:27:38,752 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,44873,1733405257343' 2024-12-05T13:27:38,753 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T13:27:38,753 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T13:27:38,754 DEBUG [RS:2;88aaa70743e7:44873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T13:27:38,754 INFO [RS:2;88aaa70743e7:44873 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T13:27:38,754 INFO [RS:2;88aaa70743e7:44873 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T13:27:38,755 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,755 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:38,755 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(1482): Serving as 88aaa70743e7,33089,1733405257168, RpcServer on 88aaa70743e7/172.17.0.3:33089, sessionid=0x101a6daed470001 2024-12-05T13:27:38,755 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1482): Serving as 88aaa70743e7,35441,1733405257297, RpcServer on 88aaa70743e7/172.17.0.3:35441, sessionid=0x101a6daed470002 2024-12-05T13:27:38,756 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T13:27:38,756 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T13:27:38,756 DEBUG [RS:0;88aaa70743e7:33089 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88aaa70743e7,33089,1733405257168 2024-12-05T13:27:38,756 DEBUG [RS:1;88aaa70743e7:35441 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88aaa70743e7,35441,1733405257297 2024-12-05T13:27:38,756 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,33089,1733405257168' 2024-12-05T13:27:38,756 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,35441,1733405257297' 2024-12-05T13:27:38,756 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T13:27:38,756 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T13:27:38,757 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T13:27:38,757 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T13:27:38,757 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T13:27:38,757 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T13:27:38,757 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T13:27:38,757 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T13:27:38,757 DEBUG [RS:1;88aaa70743e7:35441 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88aaa70743e7,35441,1733405257297 2024-12-05T13:27:38,757 DEBUG [RS:0;88aaa70743e7:33089 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88aaa70743e7,33089,1733405257168 2024-12-05T13:27:38,757 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,35441,1733405257297' 2024-12-05T13:27:38,757 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T13:27:38,757 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,33089,1733405257168' 2024-12-05T13:27:38,757 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T13:27:38,758 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T13:27:38,758 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T13:27:38,758 DEBUG [RS:1;88aaa70743e7:35441 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T13:27:38,758 INFO [RS:1;88aaa70743e7:35441 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T13:27:38,759 INFO [RS:1;88aaa70743e7:35441 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T13:27:38,759 DEBUG [RS:0;88aaa70743e7:33089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T13:27:38,759 INFO [RS:0;88aaa70743e7:33089 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T13:27:38,759 INFO [RS:0;88aaa70743e7:33089 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T13:27:38,776 WARN [88aaa70743e7:43665 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T13:27:38,859 INFO [RS:2;88aaa70743e7:44873 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:38,859 INFO [RS:1;88aaa70743e7:35441 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:38,859 INFO [RS:0;88aaa70743e7:33089 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:38,862 INFO [RS:1;88aaa70743e7:35441 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C35441%2C1733405257297, suffix=, logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs, maxLogs=32 2024-12-05T13:27:38,862 INFO [RS:2;88aaa70743e7:44873 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C44873%2C1733405257343, suffix=, logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,44873,1733405257343, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs, maxLogs=32 2024-12-05T13:27:38,863 INFO [RS:0;88aaa70743e7:33089 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C33089%2C1733405257168, suffix=, logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,33089,1733405257168, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs, maxLogs=32 2024-12-05T13:27:38,880 DEBUG [RS:2;88aaa70743e7:44873 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,44873,1733405257343/88aaa70743e7%2C44873%2C1733405257343.1733405258866, exclude list is [], retry=0 2024-12-05T13:27:38,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:38,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:38,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:38,885 DEBUG [RS:1;88aaa70743e7:35441 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297/88aaa70743e7%2C35441%2C1733405257297.1733405258866, exclude list is [], retry=0 2024-12-05T13:27:38,885 DEBUG [RS:0;88aaa70743e7:33089 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,33089,1733405257168/88aaa70743e7%2C33089%2C1733405257168.1733405258866, exclude list is [], retry=0 2024-12-05T13:27:38,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:38,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:38,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:38,892 INFO [RS:2;88aaa70743e7:44873 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,44873,1733405257343/88aaa70743e7%2C44873%2C1733405257343.1733405258866 2024-12-05T13:27:38,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:38,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:38,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:38,895 DEBUG [RS:2;88aaa70743e7:44873 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:38,897 INFO [RS:1;88aaa70743e7:35441 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297/88aaa70743e7%2C35441%2C1733405257297.1733405258866 2024-12-05T13:27:38,902 DEBUG [RS:1;88aaa70743e7:35441 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:38,903 INFO [RS:0;88aaa70743e7:33089 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,33089,1733405257168/88aaa70743e7%2C33089%2C1733405257168.1733405258866 2024-12-05T13:27:38,904 DEBUG [RS:0;88aaa70743e7:33089 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:39,028 DEBUG [88aaa70743e7:43665 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-05T13:27:39,034 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:27:39,040 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:27:39,040 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:27:39,040 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-05T13:27:39,040 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:27:39,040 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:27:39,040 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T13:27:39,040 INFO [88aaa70743e7:43665 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:27:39,040 INFO [88aaa70743e7:43665 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:27:39,040 INFO [88aaa70743e7:43665 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T13:27:39,040 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:27:39,047 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:39,052 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,35441,1733405257297, state=OPENING 2024-12-05T13:27:39,062 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T13:27:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:27:39,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,075 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:27:39,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,35441,1733405257297}] 2024-12-05T13:27:39,255 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:27:39,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36603, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:27:39,273 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T13:27:39,274 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:39,275 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T13:27:39,278 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C35441%2C1733405257297.meta, suffix=.meta, logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs, maxLogs=32 2024-12-05T13:27:39,295 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, exclude list is [], retry=0 2024-12-05T13:27:39,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:39,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:39,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:39,302 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta 2024-12-05T13:27:39,303 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:27:39,303 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:39,305 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T13:27:39,307 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T13:27:39,312 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T13:27:39,316 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T13:27:39,317 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:39,317 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T13:27:39,317 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T13:27:39,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T13:27:39,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T13:27:39,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:39,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:39,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T13:27:39,325 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T13:27:39,325 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:39,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:39,326 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T13:27:39,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T13:27:39,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:39,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:39,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T13:27:39,331 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T13:27:39,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:39,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:39,332 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T13:27:39,334 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740 2024-12-05T13:27:39,338 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740 2024-12-05T13:27:39,342 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T13:27:39,342 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T13:27:39,344 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:27:39,349 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T13:27:39,351 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69313821, jitterRate=0.03285641968250275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:27:39,352 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T13:27:39,353 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733405259318Writing region info on filesystem at 1733405259318Initializing all the Stores at 1733405259320 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405259320Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405259320Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405259320Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405259320Cleaning up temporary data from old regions at 1733405259342 (+22 ms)Running coprocessor post-open hooks at 1733405259352 (+10 ms)Region opened successfully at 1733405259353 (+1 ms) 2024-12-05T13:27:39,365 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733405259246 2024-12-05T13:27:39,378 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T13:27:39,379 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T13:27:39,381 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:39,383 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,35441,1733405257297, state=OPEN 2024-12-05T13:27:39,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:39,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:39,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:39,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:39,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,396 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:39,397 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:39,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T13:27:39,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,35441,1733405257297 in 320 msec 2024-12-05T13:27:39,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T13:27:39,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 792 msec 2024-12-05T13:27:39,412 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T13:27:39,412 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T13:27:39,428 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:27:39,429 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1] 2024-12-05T13:27:39,473 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:27:39,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:27:39,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1590 sec 2024-12-05T13:27:39,504 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733405259504, completionTime=-1 2024-12-05T13:27:39,508 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-05T13:27:39,508 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:27:39,536 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-05T13:27:39,537 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733405319537 2024-12-05T13:27:39,537 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733405379537 2024-12-05T13:27:39,537 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-05T13:27:39,538 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:27:39,546 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,43665,1733405256332-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:39,547 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,43665,1733405256332-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:39,547 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,43665,1733405256332-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:39,549 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-88aaa70743e7:43665, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:39,549 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:39,550 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:39,555 DEBUG [master/88aaa70743e7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T13:27:39,582 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.138sec 2024-12-05T13:27:39,583 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T13:27:39,584 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T13:27:39,585 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T13:27:39,586 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T13:27:39,586 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T13:27:39,586 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,43665,1733405256332-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:27:39,587 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,43665,1733405256332-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T13:27:39,591 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T13:27:39,592 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T13:27:39,593 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,43665,1733405256332-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:39,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181f9e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T13:27:39,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 88aaa70743e7,43665,-1 for getting cluster id 2024-12-05T13:27:39,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T13:27:39,694 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3e1a98ee-101f-4ab3-b69a-238955073761' 2024-12-05T13:27:39,696 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T13:27:39,697 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3e1a98ee-101f-4ab3-b69a-238955073761" 2024-12-05T13:27:39,697 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc6f80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T13:27:39,697 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [88aaa70743e7,43665,-1] 2024-12-05T13:27:39,700 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T13:27:39,702 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:27:39,703 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T13:27:39,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a2fdee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T13:27:39,707 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:27:39,715 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1] 2024-12-05T13:27:39,715 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:27:39,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:27:39,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=88aaa70743e7,43665,1733405256332 2024-12-05T13:27:39,736 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44203/hbase 2024-12-05T13:27:39,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=4474 2024-12-05T13:27:39,778 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:39,783 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:39,785 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:39,791 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45910966, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-45910966, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:39,813 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45910966/hregion-45910966.1733405259793, exclude list is [], retry=0 2024-12-05T13:27:39,819 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:39,819 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:39,820 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:39,827 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45910966/hregion-45910966.1733405259793 2024-12-05T13:27:39,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:27:39,828 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8e6deb731d3b1f22aa992344353de151, NAME => 'testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:27:39,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741838_1014 (size=64) 2024-12-05T13:27:39,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741838_1014 (size=64) 2024-12-05T13:27:39,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741838_1014 (size=64) 2024-12-05T13:27:40,244 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:40,247 INFO [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,251 INFO [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e6deb731d3b1f22aa992344353de151 columnFamilyName a 2024-12-05T13:27:40,251 DEBUG [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:40,252 INFO [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] regionserver.HStore(327): Store=8e6deb731d3b1f22aa992344353de151/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:40,252 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,254 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,254 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,255 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,255 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,258 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,262 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:40,263 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e6deb731d3b1f22aa992344353de151; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71408257, jitterRate=0.06406594812870026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:27:40,265 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e6deb731d3b1f22aa992344353de151: Writing region info on filesystem at 1733405260245Initializing all the Stores at 1733405260247 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405260247Cleaning up temporary data from old regions at 1733405260255 (+8 ms)Region opened successfully at 1733405260264 (+9 ms) 2024-12-05T13:27:40,265 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8e6deb731d3b1f22aa992344353de151, disabling compactions & flushes 2024-12-05T13:27:40,265 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151. 2024-12-05T13:27:40,265 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151. 2024-12-05T13:27:40,265 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151. after waiting 0 ms 2024-12-05T13:27:40,265 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151. 2024-12-05T13:27:40,266 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151. 2024-12-05T13:27:40,266 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8e6deb731d3b1f22aa992344353de151: Waiting for close lock at 1733405260265Disabling compacts and flushes for region at 1733405260265Disabling writes for close at 1733405260265Writing region close event to WAL at 1733405260265Closed at 1733405260266 (+1 ms) 2024-12-05T13:27:40,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741837_1013 (size=93) 2024-12-05T13:27:40,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741837_1013 (size=93) 2024-12-05T13:27:40,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741837_1013 (size=93) 2024-12-05T13:27:40,287 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:40,287 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45910966:(num 1733405259793) 2024-12-05T13:27:40,289 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-05T13:27:40,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741839_1015 (size=369) 2024-12-05T13:27:40,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741839_1015 (size=369) 2024-12-05T13:27:40,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741839_1015 (size=369) 2024-12-05T13:27:40,314 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-05T13:27:40,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741840_1016 (size=231) 2024-12-05T13:27:40,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741840_1016 (size=231) 2024-12-05T13:27:40,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741840_1016 (size=231) 2024-12-05T13:27:40,353 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1, size=369 (369bytes) 2024-12-05T13:27:40,354 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-05T13:27:40,354 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-05T13:27:40,354 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1 2024-12-05T13:27:40,360 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1 after 4ms 2024-12-05T13:27:40,368 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1 took 16ms 2024-12-05T13:27:40,372 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1 so closing down 2024-12-05T13:27:40,377 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-05T13:27:40,379 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:40,380 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000001-wal-1.temp 2024-12-05T13:27:40,381 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:40,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741841_1017 (size=369) 2024-12-05T13:27:40,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741841_1017 (size=369) 2024-12-05T13:27:40,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741841_1017 (size=369) 2024-12-05T13:27:40,407 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:40,410 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002 2024-12-05T13:27:40,416 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-05T13:27:40,416 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1, journal: Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1, size=369 (369bytes) at 1733405260353Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1 so closing down at 1733405260372 (+19 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000001-wal-1.temp at 1733405260380 (+8 ms)3 split writer threads finished at 1733405260381 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733405260407 (+26 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002 at 1733405260410 (+3 ms)Processed 2 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733405260416 (+6 ms) 2024-12-05T13:27:40,437 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2, size=231 (231bytes) 2024-12-05T13:27:40,437 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2 2024-12-05T13:27:40,439 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2 after 0ms 2024-12-05T13:27:40,443 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2 took 6ms 2024-12-05T13:27:40,447 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2 so closing down 2024-12-05T13:27:40,447 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:40,451 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-05T13:27:40,453 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002-wal-2.temp 2024-12-05T13:27:40,453 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:40,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741842_1018 (size=231) 2024-12-05T13:27:40,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741842_1018 (size=231) 2024-12-05T13:27:40,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741842_1018 (size=231) 2024-12-05T13:27:40,475 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:40,484 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-12-05T13:27:40,486 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-05T13:27:40,486 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2, journal: Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2, size=231 (231bytes) at 1733405260437Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2 so closing down at 1733405260447 (+10 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002-wal-2.temp at 1733405260453 (+6 ms)3 split writer threads finished at 1733405260453Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733405260475 (+22 ms)Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733405260486 (+11 ms) 2024-12-05T13:27:40,486 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:40,493 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:40,513 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal.1733405260493, exclude list is [], retry=0 2024-12-05T13:27:40,519 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:40,520 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:40,520 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:40,527 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal.1733405260493 2024-12-05T13:27:40,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:40,531 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e6deb731d3b1f22aa992344353de151, NAME => 'testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:40,531 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:40,531 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,531 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,539 INFO [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,543 INFO [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e6deb731d3b1f22aa992344353de151 columnFamilyName a 2024-12-05T13:27:40,543 DEBUG [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:40,546 INFO [StoreOpener-8e6deb731d3b1f22aa992344353de151-1 {}] regionserver.HStore(327): Store=8e6deb731d3b1f22aa992344353de151/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:40,546 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,547 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,551 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,553 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002 2024-12-05T13:27:40,565 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002 2024-12-05T13:27:40,569 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e6deb731d3b1f22aa992344353de151 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-05T13:27:40,630 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/.tmp/a/84e125847dc7458b955408edb7724fbc is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733405260287/Put/seqid=0 2024-12-05T13:27:40,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741844_1020 (size=5170) 2024-12-05T13:27:40,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741844_1020 (size=5170) 2024-12-05T13:27:40,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741844_1020 (size=5170) 2024-12-05T13:27:40,652 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/.tmp/a/84e125847dc7458b955408edb7724fbc 2024-12-05T13:27:40,698 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/.tmp/a/84e125847dc7458b955408edb7724fbc as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/a/84e125847dc7458b955408edb7724fbc 2024-12-05T13:27:40,709 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/a/84e125847dc7458b955408edb7724fbc, entries=2, sequenceid=2, filesize=5.0 K 2024-12-05T13:27:40,716 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8e6deb731d3b1f22aa992344353de151 in 145ms, sequenceid=2, compaction requested=false; wal=null 2024-12-05T13:27:40,719 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/0000000000000000002 2024-12-05T13:27:40,720 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,720 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,726 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e6deb731d3b1f22aa992344353de151 2024-12-05T13:27:40,731 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/8e6deb731d3b1f22aa992344353de151/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-05T13:27:40,732 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e6deb731d3b1f22aa992344353de151; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60868427, jitterRate=-0.09298975765705109}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:27:40,734 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e6deb731d3b1f22aa992344353de151: Writing region info on filesystem at 1733405260532Initializing all the Stores at 1733405260534 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405260534Obtaining lock to block concurrent updates at 1733405260569 (+35 ms)Preparing flush snapshotting stores in 8e6deb731d3b1f22aa992344353de151 at 1733405260569Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733405260573 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733405259779.8e6deb731d3b1f22aa992344353de151. at 1733405260573Flushing 8e6deb731d3b1f22aa992344353de151/a: creating writer at 1733405260575 (+2 ms)Flushing 8e6deb731d3b1f22aa992344353de151/a: appending metadata at 1733405260618 (+43 ms)Flushing 8e6deb731d3b1f22aa992344353de151/a: closing flushed file at 1733405260622 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ffa7a8b: reopening flushed file at 1733405260696 (+74 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8e6deb731d3b1f22aa992344353de151 in 145ms, sequenceid=2, compaction requested=false; wal=null at 1733405260717 (+21 ms)Cleaning up temporary data from old regions at 1733405260720 (+3 ms)Region opened successfully at 1733405260733 (+13 ms) 2024-12-05T13:27:40,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 365) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:48868 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44203/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:48870 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:52686 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:52528 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:58680 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:58776 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 215), ProcessCount=11 (was 11), AvailableMemoryMB=5006 (was 4474) - AvailableMemoryMB LEAK? - 2024-12-05T13:27:40,782 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=5006 2024-12-05T13:27:40,799 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:40,803 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:40,806 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:40,812 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55787840, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-55787840, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:40,831 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55787840/hregion-55787840.1733405260812, exclude list is [], retry=0 2024-12-05T13:27:40,837 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:40,838 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:40,839 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:40,852 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55787840/hregion-55787840.1733405260812 2024-12-05T13:27:40,855 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:27:40,855 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e1db4266da91e8de62cac1fb634eff40, NAME => 'testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:27:40,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741846_1022 (size=64) 2024-12-05T13:27:40,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741846_1022 (size=64) 2024-12-05T13:27:40,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741846_1022 (size=64) 2024-12-05T13:27:40,884 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:40,887 INFO [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:40,896 INFO [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1db4266da91e8de62cac1fb634eff40 columnFamilyName a 2024-12-05T13:27:40,896 DEBUG [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:40,898 INFO [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] regionserver.HStore(327): Store=e1db4266da91e8de62cac1fb634eff40/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:40,898 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:40,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:40,902 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:40,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:40,904 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:40,909 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:40,915 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:40,917 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e1db4266da91e8de62cac1fb634eff40; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70418368, jitterRate=0.049315452575683594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:27:40,917 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e1db4266da91e8de62cac1fb634eff40: Writing region info on filesystem at 1733405260884Initializing all the Stores at 1733405260886 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405260886Cleaning up temporary data from old regions at 1733405260904 (+18 ms)Region opened successfully at 1733405260917 (+13 ms) 2024-12-05T13:27:40,917 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e1db4266da91e8de62cac1fb634eff40, disabling compactions & flushes 2024-12-05T13:27:40,918 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40. 2024-12-05T13:27:40,918 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40. 2024-12-05T13:27:40,918 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40. after waiting 0 ms 2024-12-05T13:27:40,918 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40. 2024-12-05T13:27:40,923 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40. 2024-12-05T13:27:40,923 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e1db4266da91e8de62cac1fb634eff40: Waiting for close lock at 1733405260917Disabling compacts and flushes for region at 1733405260917Disabling writes for close at 1733405260918 (+1 ms)Writing region close event to WAL at 1733405260923 (+5 ms)Closed at 1733405260923 2024-12-05T13:27:40,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741845_1021 (size=93) 2024-12-05T13:27:40,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741845_1021 (size=93) 2024-12-05T13:27:40,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741845_1021 (size=93) 2024-12-05T13:27:40,940 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:40,940 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55787840:(num 1733405260812) 2024-12-05T13:27:40,942 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-05T13:27:40,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741847_1023 (size=369) 2024-12-05T13:27:40,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741847_1023 (size=369) 2024-12-05T13:27:40,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741847_1023 (size=369) 2024-12-05T13:27:40,978 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-05T13:27:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741848_1024 (size=231) 2024-12-05T13:27:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741848_1024 (size=231) 2024-12-05T13:27:41,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741848_1024 (size=231) 2024-12-05T13:27:41,033 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2, size=231 (231bytes) 2024-12-05T13:27:41,033 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2 2024-12-05T13:27:41,034 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2 after 1ms 2024-12-05T13:27:41,039 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2 took 7ms 2024-12-05T13:27:41,043 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2 so closing down 2024-12-05T13:27:41,043 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:41,046 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-05T13:27:41,048 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002-wal-2.temp 2024-12-05T13:27:41,049 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:41,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741849_1025 (size=231) 2024-12-05T13:27:41,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741849_1025 (size=231) 2024-12-05T13:27:41,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741849_1025 (size=231) 2024-12-05T13:27:41,069 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:41,073 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002 2024-12-05T13:27:41,073 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-12-05T13:27:41,073 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2, journal: Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2, size=231 (231bytes) at 1733405261033Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2 so closing down at 1733405261043 (+10 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002-wal-2.temp at 1733405261049 (+6 ms)3 split writer threads finished at 1733405261049Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733405261070 (+21 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002 at 1733405261073 (+3 ms)Processed 1 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1733405261073 2024-12-05T13:27:41,094 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1, size=369 (369bytes) 2024-12-05T13:27:41,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1 2024-12-05T13:27:41,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1 after 0ms 2024-12-05T13:27:41,101 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1 took 7ms 2024-12-05T13:27:41,105 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1 so closing down 2024-12-05T13:27:41,107 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:41,110 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-05T13:27:41,112 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000001-wal-1.temp 2024-12-05T13:27:41,112 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:41,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741850_1026 (size=369) 2024-12-05T13:27:41,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741850_1026 (size=369) 2024-12-05T13:27:41,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741850_1026 (size=369) 2024-12-05T13:27:41,131 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:41,139 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002, length=231 2024-12-05T13:27:41,142 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002 2024-12-05T13:27:41,142 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-12-05T13:27:41,142 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1, journal: Splitting hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1, size=369 (369bytes) at 1733405261094Finishing writing output for hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1 so closing down at 1733405261107 (+13 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000001-wal-1.temp at 1733405261112 (+5 ms)3 split writer threads finished at 1733405261112Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733405261132 (+20 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002 at 1733405261142 (+10 ms)Processed 2 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1733405261142 2024-12-05T13:27:41,142 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:41,145 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:41,160 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal.1733405261146, exclude list is [], retry=0 2024-12-05T13:27:41,164 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:41,164 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:41,164 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:41,167 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal.1733405261146 2024-12-05T13:27:41,167 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:41,167 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e1db4266da91e8de62cac1fb634eff40, NAME => 'testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:41,167 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:41,167 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,167 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,171 INFO [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,173 INFO [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1db4266da91e8de62cac1fb634eff40 columnFamilyName a 2024-12-05T13:27:41,173 DEBUG [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:41,174 INFO [StoreOpener-e1db4266da91e8de62cac1fb634eff40-1 {}] regionserver.HStore(327): Store=e1db4266da91e8de62cac1fb634eff40/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:41,174 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,178 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,179 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002 2024-12-05T13:27:41,183 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002 2024-12-05T13:27:41,184 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e1db4266da91e8de62cac1fb634eff40 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-05T13:27:41,202 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/.tmp/a/cb21db57d3e54b10a9fc468d52b090b4 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733405260940/Put/seqid=0 2024-12-05T13:27:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741852_1028 (size=5170) 2024-12-05T13:27:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741852_1028 (size=5170) 2024-12-05T13:27:41,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741852_1028 (size=5170) 2024-12-05T13:27:41,223 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/.tmp/a/cb21db57d3e54b10a9fc468d52b090b4 2024-12-05T13:27:41,234 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/.tmp/a/cb21db57d3e54b10a9fc468d52b090b4 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/a/cb21db57d3e54b10a9fc468d52b090b4 2024-12-05T13:27:41,245 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/a/cb21db57d3e54b10a9fc468d52b090b4, entries=2, sequenceid=2, filesize=5.0 K 2024-12-05T13:27:41,246 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e1db4266da91e8de62cac1fb634eff40 in 62ms, sequenceid=2, compaction requested=false; wal=null 2024-12-05T13:27:41,247 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/0000000000000000002 2024-12-05T13:27:41,248 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,248 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,251 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e1db4266da91e8de62cac1fb634eff40 2024-12-05T13:27:41,258 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/e1db4266da91e8de62cac1fb634eff40/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-05T13:27:41,260 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e1db4266da91e8de62cac1fb634eff40; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70451642, jitterRate=0.04981127381324768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:27:41,260 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e1db4266da91e8de62cac1fb634eff40: Writing region info on filesystem at 1733405261167Initializing all the Stores at 1733405261169 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405261169Obtaining lock to block concurrent updates at 1733405261184 (+15 ms)Preparing flush snapshotting stores in e1db4266da91e8de62cac1fb634eff40 at 1733405261184Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733405261184Flushing stores of testReplayEditsWrittenIntoWAL,,1733405260800.e1db4266da91e8de62cac1fb634eff40. at 1733405261184Flushing e1db4266da91e8de62cac1fb634eff40/a: creating writer at 1733405261184Flushing e1db4266da91e8de62cac1fb634eff40/a: appending metadata at 1733405261201 (+17 ms)Flushing e1db4266da91e8de62cac1fb634eff40/a: closing flushed file at 1733405261201Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a16ae71: reopening flushed file at 1733405261233 (+32 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e1db4266da91e8de62cac1fb634eff40 in 62ms, sequenceid=2, compaction requested=false; wal=null at 1733405261246 (+13 ms)Cleaning up temporary data from old regions at 1733405261248 (+2 ms)Region opened successfully at 1733405261260 (+12 ms) 2024-12-05T13:27:41,280 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:58844 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:48870 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:52742 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:52528 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:58680 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:48932 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 215), ProcessCount=11 (was 11), AvailableMemoryMB=4979 (was 5006) 2024-12-05T13:27:41,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=215, ProcessCount=11, AvailableMemoryMB=4978 2024-12-05T13:27:41,315 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:41,317 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:41,319 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:41,324 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59233776, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-59233776, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:41,340 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59233776/hregion-59233776.1733405261325, exclude list is [], retry=0 2024-12-05T13:27:41,344 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:41,344 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:41,344 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:41,348 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59233776/hregion-59233776.1733405261325 2024-12-05T13:27:41,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:41,355 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => edebb302306b6a134057ad8b8f936a09, NAME => 'testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:27:41,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741854_1030 (size=64) 2024-12-05T13:27:41,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741854_1030 (size=64) 2024-12-05T13:27:41,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741854_1030 (size=64) 2024-12-05T13:27:41,371 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:41,373 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,375 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edebb302306b6a134057ad8b8f936a09 columnFamilyName a 2024-12-05T13:27:41,375 DEBUG [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:41,376 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(327): Store=edebb302306b6a134057ad8b8f936a09/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:41,376 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,379 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edebb302306b6a134057ad8b8f936a09 columnFamilyName b 2024-12-05T13:27:41,379 DEBUG [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:41,379 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(327): Store=edebb302306b6a134057ad8b8f936a09/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:41,380 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,381 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edebb302306b6a134057ad8b8f936a09 columnFamilyName c 2024-12-05T13:27:41,382 DEBUG [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:41,382 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(327): Store=edebb302306b6a134057ad8b8f936a09/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:41,383 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,384 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,387 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,387 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,388 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:41,390 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:41,393 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:41,394 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened edebb302306b6a134057ad8b8f936a09; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68021205, jitterRate=0.013594940304756165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:41,395 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for edebb302306b6a134057ad8b8f936a09: Writing region info on filesystem at 1733405261371Initializing all the Stores at 1733405261372 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405261372Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405261372Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405261372Cleaning up temporary data from old regions at 1733405261387 (+15 ms)Region opened successfully at 1733405261394 (+7 ms) 2024-12-05T13:27:41,395 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing edebb302306b6a134057ad8b8f936a09, disabling compactions & flushes 2024-12-05T13:27:41,395 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:41,395 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:41,395 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. after waiting 0 ms 2024-12-05T13:27:41,395 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:41,396 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:41,396 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for edebb302306b6a134057ad8b8f936a09: Waiting for close lock at 1733405261395Disabling compacts and flushes for region at 1733405261395Disabling writes for close at 1733405261395Writing region close event to WAL at 1733405261396 (+1 ms)Closed at 1733405261396 2024-12-05T13:27:41,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741853_1029 (size=93) 2024-12-05T13:27:41,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741853_1029 (size=93) 2024-12-05T13:27:41,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741853_1029 (size=93) 2024-12-05T13:27:41,403 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:41,403 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59233776:(num 1733405261325) 2024-12-05T13:27:41,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:41,406 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:41,422 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406, exclude list is [], retry=0 2024-12-05T13:27:41,426 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:41,427 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:41,428 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:41,433 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 2024-12-05T13:27:41,434 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:41,646 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406, size=0 (0bytes) 2024-12-05T13:27:41,646 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 might be still open, length is 0 2024-12-05T13:27:41,646 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 2024-12-05T13:27:41,647 WARN [IPC Server handler 0 on default port 44203 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-05T13:27:41,648 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 after 2ms 2024-12-05T13:27:44,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:48956 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48956 dst: /127.0.0.1:40037 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40037 remote=/127.0.0.1:48956]. Total timeout mills is 60000, 57111 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:44,498 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:52776 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38677:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52776 dst: /127.0.0.1:38677 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:44,499 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:58874 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58874 dst: /127.0.0.1:43089 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:44,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741855_1032 (size=470675) 2024-12-05T13:27:44,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741855_1032 (size=470675) 2024-12-05T13:27:44,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741855_1032 (size=470675) 2024-12-05T13:27:44,830 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T13:27:44,885 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T13:27:45,649 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 after 4003ms 2024-12-05T13:27:45,655 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 took 4009ms 2024-12-05T13:27:45,664 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733405261406.temp 2024-12-05T13:27:45,666 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000000001-wal.1733405261406.temp 2024-12-05T13:27:45,731 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406; continuing. 2024-12-05T13:27:45,731 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 so closing down 2024-12-05T13:27:45,731 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:45,732 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:45,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741856_1033 (size=470683) 2024-12-05T13:27:45,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741856_1033 (size=470683) 2024-12-05T13:27:45,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741856_1033 (size=470683) 2024-12-05T13:27:45,735 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000000001-wal.1733405261406.temp (wrote 3002 edits, skipped 0 edits in 29 ms) 2024-12-05T13:27:45,738 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000000001-wal.1733405261406.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000003002 2024-12-05T13:27:45,738 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406, size=0, length=0, corrupted=false, cancelled=false 2024-12-05T13:27:45,739 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406, journal: Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406, size=0 (0bytes) at 1733405261646Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000000001-wal.1733405261406.temp at 1733405265667 (+4021 ms)Split 1024 edits, skipped 0 edits. at 1733405265701 (+34 ms)Split 2048 edits, skipped 0 edits. at 1733405265718 (+17 ms)Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 so closing down at 1733405265731 (+13 ms)3 split writer threads finished at 1733405265732 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000000001-wal.1733405261406.temp (wrote 3002 edits, skipped 0 edits in 29 ms) at 1733405265735 (+3 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000000001-wal.1733405261406.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000003002 at 1733405265738 (+3 ms)Processed 3002 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406, size=0, length=0, corrupted=false, cancelled=false at 1733405265738 2024-12-05T13:27:45,742 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405261406 2024-12-05T13:27:45,743 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000003002 2024-12-05T13:27:45,744 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:45,746 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:45,762 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405265747, exclude list is [], retry=0 2024-12-05T13:27:45,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:45,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:45,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:45,768 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405265747 2024-12-05T13:27:45,769 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:45,769 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:45,771 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:45,774 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edebb302306b6a134057ad8b8f936a09 columnFamilyName a 2024-12-05T13:27:45,774 DEBUG [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:45,775 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(327): Store=edebb302306b6a134057ad8b8f936a09/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:45,775 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:45,776 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edebb302306b6a134057ad8b8f936a09 columnFamilyName b 2024-12-05T13:27:45,776 DEBUG [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:45,776 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(327): Store=edebb302306b6a134057ad8b8f936a09/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:45,776 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:45,777 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edebb302306b6a134057ad8b8f936a09 columnFamilyName c 2024-12-05T13:27:45,778 DEBUG [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:45,778 INFO [StoreOpener-edebb302306b6a134057ad8b8f936a09-1 {}] regionserver.HStore(327): Store=edebb302306b6a134057ad8b8f936a09/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:45,778 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:45,779 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:45,781 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:45,783 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000003002 2024-12-05T13:27:45,820 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-05T13:27:46,231 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing edebb302306b6a134057ad8b8f936a09 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-05T13:27:46,274 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/a/bb36699ecb9448e49f3fd9e105905808 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733405261446/Put/seqid=0 2024-12-05T13:27:46,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741858_1035 (size=50463) 2024-12-05T13:27:46,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741858_1035 (size=50463) 2024-12-05T13:27:46,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741858_1035 (size=50463) 2024-12-05T13:27:46,283 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/a/bb36699ecb9448e49f3fd9e105905808 2024-12-05T13:27:46,292 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/a/bb36699ecb9448e49f3fd9e105905808 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/a/bb36699ecb9448e49f3fd9e105905808 2024-12-05T13:27:46,300 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/a/bb36699ecb9448e49f3fd9e105905808, entries=754, sequenceid=754, filesize=49.3 K 2024-12-05T13:27:46,300 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for edebb302306b6a134057ad8b8f936a09 in 69ms, sequenceid=754, compaction requested=false; wal=null 2024-12-05T13:27:46,318 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-05T13:27:46,318 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing edebb302306b6a134057ad8b8f936a09 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-05T13:27:46,332 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/a/7c384f9912b6460d87bda2d638e9b4cb is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733405261493/Put/seqid=0 2024-12-05T13:27:46,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741859_1036 (size=20072) 2024-12-05T13:27:46,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741859_1036 (size=20072) 2024-12-05T13:27:46,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741859_1036 (size=20072) 2024-12-05T13:27:46,346 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/a/7c384f9912b6460d87bda2d638e9b4cb 2024-12-05T13:27:46,376 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/b/ea4335c3f64c4493933413ff6081d0a5 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733405261516/Put/seqid=0 2024-12-05T13:27:46,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741860_1037 (size=35835) 2024-12-05T13:27:46,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741860_1037 (size=35835) 2024-12-05T13:27:46,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741860_1037 (size=35835) 2024-12-05T13:27:46,389 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/b/ea4335c3f64c4493933413ff6081d0a5 2024-12-05T13:27:46,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/a/7c384f9912b6460d87bda2d638e9b4cb as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/a/7c384f9912b6460d87bda2d638e9b4cb 2024-12-05T13:27:46,406 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/a/7c384f9912b6460d87bda2d638e9b4cb, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-05T13:27:46,410 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/b/ea4335c3f64c4493933413ff6081d0a5 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/b/ea4335c3f64c4493933413ff6081d0a5 2024-12-05T13:27:46,417 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/b/ea4335c3f64c4493933413ff6081d0a5, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-05T13:27:46,418 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for edebb302306b6a134057ad8b8f936a09 in 99ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-05T13:27:46,430 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-05T13:27:46,430 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing edebb302306b6a134057ad8b8f936a09 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-05T13:27:46,439 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/b/f07e58904b8f45929cbc31988909ab50 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733405261534/Put/seqid=0 2024-12-05T13:27:46,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741861_1038 (size=35082) 2024-12-05T13:27:46,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741861_1038 (size=35082) 2024-12-05T13:27:46,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741861_1038 (size=35082) 2024-12-05T13:27:46,450 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/b/f07e58904b8f45929cbc31988909ab50 2024-12-05T13:27:46,478 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/c/8d408e452ca149cf8c87a734315659b8 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733405261558/Put/seqid=0 2024-12-05T13:27:46,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741862_1039 (size=20825) 2024-12-05T13:27:46,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741862_1039 (size=20825) 2024-12-05T13:27:46,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741862_1039 (size=20825) 2024-12-05T13:27:46,490 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/c/8d408e452ca149cf8c87a734315659b8 2024-12-05T13:27:46,498 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/b/f07e58904b8f45929cbc31988909ab50 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/b/f07e58904b8f45929cbc31988909ab50 2024-12-05T13:27:46,506 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/b/f07e58904b8f45929cbc31988909ab50, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-05T13:27:46,508 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/c/8d408e452ca149cf8c87a734315659b8 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/c/8d408e452ca149cf8c87a734315659b8 2024-12-05T13:27:46,515 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/c/8d408e452ca149cf8c87a734315659b8, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-05T13:27:46,516 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for edebb302306b6a134057ad8b8f936a09 in 86ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-05T13:27:46,525 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733405261606/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:46,528 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000003002 2024-12-05T13:27:46,528 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-05T13:27:46,528 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing edebb302306b6a134057ad8b8f936a09 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-05T13:27:46,538 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/c/b05985b957be485b85855fab581fef6f is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733405261564/Put/seqid=0 2024-12-05T13:27:46,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741863_1040 (size=50301) 2024-12-05T13:27:46,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741863_1040 (size=50301) 2024-12-05T13:27:46,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741863_1040 (size=50301) 2024-12-05T13:27:46,552 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/c/b05985b957be485b85855fab581fef6f 2024-12-05T13:27:46,559 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b05985b957be485b85855fab581fef6f 2024-12-05T13:27:46,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/.tmp/c/b05985b957be485b85855fab581fef6f as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/c/b05985b957be485b85855fab581fef6f 2024-12-05T13:27:46,568 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b05985b957be485b85855fab581fef6f 2024-12-05T13:27:46,568 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/c/b05985b957be485b85855fab581fef6f, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-05T13:27:46,568 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for edebb302306b6a134057ad8b8f936a09 in 40ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-05T13:27:46,569 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/0000000000000003002 2024-12-05T13:27:46,570 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:46,571 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:46,571 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T13:27:46,573 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for edebb302306b6a134057ad8b8f936a09 2024-12-05T13:27:46,576 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenIntoWAL/edebb302306b6a134057ad8b8f936a09/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-05T13:27:46,577 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened edebb302306b6a134057ad8b8f936a09; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74239729, jitterRate=0.10625816881656647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T13:27:46,577 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for edebb302306b6a134057ad8b8f936a09: Writing region info on filesystem at 1733405265769Initializing all the Stores at 1733405265771 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405265771Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405265771Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405265771Cleaning up temporary data from old regions at 1733405266571 (+800 ms)Region opened successfully at 1733405266577 (+6 ms) 2024-12-05T13:27:46,632 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing edebb302306b6a134057ad8b8f936a09, disabling compactions & flushes 2024-12-05T13:27:46,632 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:46,632 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:46,632 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. after waiting 0 ms 2024-12-05T13:27:46,632 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:46,642 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733405261315.edebb302306b6a134057ad8b8f936a09. 2024-12-05T13:27:46,642 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for edebb302306b6a134057ad8b8f936a09: Waiting for close lock at 1733405266632Disabling compacts and flushes for region at 1733405266632Disabling writes for close at 1733405266632Writing region close event to WAL at 1733405266642 (+10 ms)Closed at 1733405266642 2024-12-05T13:27:46,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741857_1034 (size=93) 2024-12-05T13:27:46,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741857_1034 (size=93) 2024-12-05T13:27:46,653 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405265747 not finished, retry = 0 2024-12-05T13:27:46,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741857_1034 (size=93) 2024-12-05T13:27:46,670 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T13:27:46,670 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T13:27:46,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T13:27:46,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T13:27:46,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T13:27:46,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T13:27:46,673 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-05T13:27:46,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-05T13:27:46,757 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:46,757 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733405265747) 2024-12-05T13:27:46,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42385 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:44203 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44203 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:42385 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36661 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1515765489_22 at /127.0.0.1:49218 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1515765489_22 at /127.0.0.1:36070 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1515765489_22 at /127.0.0.1:32930 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:36661 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1515765489_22 at /127.0.0.1:32910 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=863 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 215) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=4666 (was 4978) 2024-12-05T13:27:46,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=400, OpenFileDescriptor=863, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=4666 2024-12-05T13:27:46,797 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:46,800 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:46,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:46,805 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23558907, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-23558907, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:46,820 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23558907/hregion-23558907.1733405266805, exclude list is [], retry=0 2024-12-05T13:27:46,823 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:46,824 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:46,825 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:46,829 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23558907/hregion-23558907.1733405266805 2024-12-05T13:27:46,830 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:46,830 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4136a7e4b93de76fd0abc63fa3157b7b, NAME => 'test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:27:46,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741865_1042 (size=43) 2024-12-05T13:27:46,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741865_1042 (size=43) 2024-12-05T13:27:46,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741865_1042 (size=43) 2024-12-05T13:27:46,850 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:46,852 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,854 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4136a7e4b93de76fd0abc63fa3157b7b columnFamilyName a 2024-12-05T13:27:46,854 DEBUG [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:46,854 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(327): Store=4136a7e4b93de76fd0abc63fa3157b7b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:46,855 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,856 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4136a7e4b93de76fd0abc63fa3157b7b columnFamilyName b 2024-12-05T13:27:46,857 DEBUG [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:46,857 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(327): Store=4136a7e4b93de76fd0abc63fa3157b7b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:46,857 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,859 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4136a7e4b93de76fd0abc63fa3157b7b columnFamilyName c 2024-12-05T13:27:46,859 DEBUG [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:46,860 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(327): Store=4136a7e4b93de76fd0abc63fa3157b7b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:46,860 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,861 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,861 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,863 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,863 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,864 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:46,866 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:46,871 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:46,872 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4136a7e4b93de76fd0abc63fa3157b7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68738697, jitterRate=0.024286404252052307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:46,874 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4136a7e4b93de76fd0abc63fa3157b7b: Writing region info on filesystem at 1733405266850Initializing all the Stores at 1733405266851 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405266851Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405266852 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405266852Cleaning up temporary data from old regions at 1733405266863 (+11 ms)Region opened successfully at 1733405266873 (+10 ms) 2024-12-05T13:27:46,874 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4136a7e4b93de76fd0abc63fa3157b7b, disabling compactions & flushes 2024-12-05T13:27:46,874 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:46,874 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:46,874 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. after waiting 0 ms 2024-12-05T13:27:46,874 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:46,875 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:46,875 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4136a7e4b93de76fd0abc63fa3157b7b: Waiting for close lock at 1733405266874Disabling compacts and flushes for region at 1733405266874Disabling writes for close at 1733405266874Writing region close event to WAL at 1733405266874Closed at 1733405266874 2024-12-05T13:27:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741864_1041 (size=93) 2024-12-05T13:27:46,887 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-23558907/hregion-23558907.1733405266805 not finished, retry = 0 2024-12-05T13:27:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741864_1041 (size=93) 2024-12-05T13:27:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741864_1041 (size=93) 2024-12-05T13:27:46,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:46,990 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23558907:(num 1733405266805) 2024-12-05T13:27:46,991 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:46,993 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:47,010 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993, exclude list is [], retry=0 2024-12-05T13:27:47,014 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:47,015 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:47,015 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:47,019 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993 2024-12-05T13:27:47,019 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741866_1043 (size=344306) 2024-12-05T13:27:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741866_1043 (size=344306) 2024-12-05T13:27:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741866_1043 (size=344306) 2024-12-05T13:27:47,245 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993, size=336.2 K (344306bytes) 2024-12-05T13:27:47,246 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993 2024-12-05T13:27:47,246 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993 after 0ms 2024-12-05T13:27:47,250 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993 took 5ms 2024-12-05T13:27:47,261 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733405266993.temp 2024-12-05T13:27:47,262 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000000001-wal.1733405266993.temp 2024-12-05T13:27:47,290 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993 so closing down 2024-12-05T13:27:47,290 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:47,290 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:47,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741867_1044 (size=344306) 2024-12-05T13:27:47,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741867_1044 (size=344306) 2024-12-05T13:27:47,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741867_1044 (size=344306) 2024-12-05T13:27:47,294 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000000001-wal.1733405266993.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-12-05T13:27:47,296 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000000001-wal.1733405266993.temp to hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003000 2024-12-05T13:27:47,296 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-12-05T13:27:47,296 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993, journal: Splitting hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993, size=336.2 K (344306bytes) at 1733405267246Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000000001-wal.1733405266993.temp at 1733405267262 (+16 ms)Split 1024 edits, skipped 0 edits. at 1733405267266 (+4 ms)Split 2048 edits, skipped 0 edits. at 1733405267279 (+13 ms)Finishing writing output for hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993 so closing down at 1733405267290 (+11 ms)3 split writer threads finished at 1733405267290Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000000001-wal.1733405266993.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1733405267294 (+4 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000000001-wal.1733405266993.temp to hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003000 at 1733405267296 (+2 ms)Processed 3000 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1733405267296 2024-12-05T13:27:47,298 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405266993 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405266993 2024-12-05T13:27:47,299 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003000 2024-12-05T13:27:47,299 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:47,302 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:47,316 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303, exclude list is [], retry=0 2024-12-05T13:27:47,320 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:47,320 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:47,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:47,325 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303 2024-12-05T13:27:47,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:27:47,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741868_1045 (size=344433) 2024-12-05T13:27:47,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741868_1045 (size=344433) 2024-12-05T13:27:47,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741868_1045 (size=344433) 2024-12-05T13:27:47,539 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303, size=336.4 K (344433bytes) 2024-12-05T13:27:47,539 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303 2024-12-05T13:27:47,540 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303 after 1ms 2024-12-05T13:27:47,545 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303 took 6ms 2024-12-05T13:27:47,552 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733405267303.temp 2024-12-05T13:27:47,559 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003001-wal.1733405267303.temp 2024-12-05T13:27:47,581 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303 so closing down 2024-12-05T13:27:47,581 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:47,581 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:47,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741869_1046 (size=344433) 2024-12-05T13:27:47,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741869_1046 (size=344433) 2024-12-05T13:27:47,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741869_1046 (size=344433) 2024-12-05T13:27:47,599 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003001-wal.1733405267303.temp (wrote 3000 edits, skipped 0 edits in 9 ms) 2024-12-05T13:27:47,601 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003001-wal.1733405267303.temp to hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000006000 2024-12-05T13:27:47,601 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 55 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-12-05T13:27:47,601 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303, journal: Splitting hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303, size=336.4 K (344433bytes) at 1733405267539Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003001-wal.1733405267303.temp at 1733405267559 (+20 ms)Split 1024 edits, skipped 0 edits. at 1733405267560 (+1 ms)Split 2048 edits, skipped 0 edits. at 1733405267571 (+11 ms)Finishing writing output for hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303 so closing down at 1733405267581 (+10 ms)3 split writer threads finished at 1733405267581Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003001-wal.1733405267303.temp (wrote 3000 edits, skipped 0 edits in 9 ms) at 1733405267599 (+18 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003001-wal.1733405267303.temp to hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000006000 at 1733405267601 (+2 ms)Processed 3000 edits across 1 Regions in 55 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1733405267601 2024-12-05T13:27:47,604 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267303 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405267303 2024-12-05T13:27:47,605 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000006000 2024-12-05T13:27:47,605 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:47,608 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/test2727-manual,16010,1733405266796, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:47,623 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267608, exclude list is [], retry=0 2024-12-05T13:27:47,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:47,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:47,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:47,629 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733405266796/wal.1733405267608 2024-12-05T13:27:47,630 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:47,630 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4136a7e4b93de76fd0abc63fa3157b7b, NAME => 'test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:47,630 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:47,631 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,631 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,635 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,637 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4136a7e4b93de76fd0abc63fa3157b7b columnFamilyName a 2024-12-05T13:27:47,637 DEBUG [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:47,638 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(327): Store=4136a7e4b93de76fd0abc63fa3157b7b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:47,638 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,640 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4136a7e4b93de76fd0abc63fa3157b7b columnFamilyName b 2024-12-05T13:27:47,640 DEBUG [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:47,641 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(327): Store=4136a7e4b93de76fd0abc63fa3157b7b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:47,641 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,642 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4136a7e4b93de76fd0abc63fa3157b7b columnFamilyName c 2024-12-05T13:27:47,643 DEBUG [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:47,644 INFO [StoreOpener-4136a7e4b93de76fd0abc63fa3157b7b-1 {}] regionserver.HStore(327): Store=4136a7e4b93de76fd0abc63fa3157b7b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:47,644 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,645 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:47,649 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003000 2024-12-05T13:27:47,682 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003000 2024-12-05T13:27:47,683 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000006000 2024-12-05T13:27:47,723 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000006000 2024-12-05T13:27:47,723 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4136a7e4b93de76fd0abc63fa3157b7b 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-05T13:27:47,752 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/a/16fe61bf7cb74db8ad7f928887431255 is 41, key is test2727/a:100/1733405267330/Put/seqid=0 2024-12-05T13:27:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741871_1048 (size=84227) 2024-12-05T13:27:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741871_1048 (size=84227) 2024-12-05T13:27:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741871_1048 (size=84227) 2024-12-05T13:27:48,167 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/a/16fe61bf7cb74db8ad7f928887431255 2024-12-05T13:27:48,202 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/b/e6cac441c4ba47ae9158dee18a3116be is 41, key is test2727/b:100/1733405267393/Put/seqid=0 2024-12-05T13:27:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741872_1049 (size=84609) 2024-12-05T13:27:48,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741872_1049 (size=84609) 2024-12-05T13:27:48,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741872_1049 (size=84609) 2024-12-05T13:27:48,209 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/b/e6cac441c4ba47ae9158dee18a3116be 2024-12-05T13:27:48,234 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/c/6c94dbc3727947ce9db3b34bf0fc5a69 is 41, key is test2727/c:100/1733405267457/Put/seqid=0 2024-12-05T13:27:48,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741873_1050 (size=84609) 2024-12-05T13:27:48,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741873_1050 (size=84609) 2024-12-05T13:27:48,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741873_1050 (size=84609) 2024-12-05T13:27:48,246 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/c/6c94dbc3727947ce9db3b34bf0fc5a69 2024-12-05T13:27:48,254 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/a/16fe61bf7cb74db8ad7f928887431255 as hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/a/16fe61bf7cb74db8ad7f928887431255 2024-12-05T13:27:48,263 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/a/16fe61bf7cb74db8ad7f928887431255, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-05T13:27:48,264 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/b/e6cac441c4ba47ae9158dee18a3116be as hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/b/e6cac441c4ba47ae9158dee18a3116be 2024-12-05T13:27:48,273 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/b/e6cac441c4ba47ae9158dee18a3116be, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-05T13:27:48,275 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/.tmp/c/6c94dbc3727947ce9db3b34bf0fc5a69 as hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/c/6c94dbc3727947ce9db3b34bf0fc5a69 2024-12-05T13:27:48,283 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/c/6c94dbc3727947ce9db3b34bf0fc5a69, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-05T13:27:48,284 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4136a7e4b93de76fd0abc63fa3157b7b in 560ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-05T13:27:48,285 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000003000 2024-12-05T13:27:48,286 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/0000000000000006000 2024-12-05T13:27:48,287 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:48,287 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:48,288 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:48,290 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4136a7e4b93de76fd0abc63fa3157b7b 2024-12-05T13:27:48,295 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/test2727/4136a7e4b93de76fd0abc63fa3157b7b/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-05T13:27:48,297 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4136a7e4b93de76fd0abc63fa3157b7b; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69138391, jitterRate=0.030242308974266052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:48,298 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4136a7e4b93de76fd0abc63fa3157b7b: Writing region info on filesystem at 1733405267631Initializing all the Stores at 1733405267632 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405267632Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405267634 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405267634Obtaining lock to block concurrent updates at 1733405267723 (+89 ms)Preparing flush snapshotting stores in 4136a7e4b93de76fd0abc63fa3157b7b at 1733405267723Finished memstore snapshotting test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733405267724 (+1 ms)Flushing stores of test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. at 1733405267724Flushing 4136a7e4b93de76fd0abc63fa3157b7b/a: creating writer at 1733405267724Flushing 4136a7e4b93de76fd0abc63fa3157b7b/a: appending metadata at 1733405267751 (+27 ms)Flushing 4136a7e4b93de76fd0abc63fa3157b7b/a: closing flushed file at 1733405267751Flushing 4136a7e4b93de76fd0abc63fa3157b7b/b: creating writer at 1733405268174 (+423 ms)Flushing 4136a7e4b93de76fd0abc63fa3157b7b/b: appending metadata at 1733405268200 (+26 ms)Flushing 4136a7e4b93de76fd0abc63fa3157b7b/b: closing flushed file at 1733405268200Flushing 4136a7e4b93de76fd0abc63fa3157b7b/c: creating writer at 1733405268215 (+15 ms)Flushing 4136a7e4b93de76fd0abc63fa3157b7b/c: appending metadata at 1733405268233 (+18 ms)Flushing 4136a7e4b93de76fd0abc63fa3157b7b/c: closing flushed file at 1733405268233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@390a6e49: reopening flushed file at 1733405268253 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@342b8571: reopening flushed file at 1733405268263 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e00ea2: reopening flushed file at 1733405268273 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4136a7e4b93de76fd0abc63fa3157b7b in 560ms, sequenceid=6000, compaction requested=false; wal=null at 1733405268284 (+11 ms)Cleaning up temporary data from old regions at 1733405268287 (+3 ms)Region opened successfully at 1733405268297 (+10 ms) 2024-12-05T13:27:48,299 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-05T13:27:48,299 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4136a7e4b93de76fd0abc63fa3157b7b, disabling compactions & flushes 2024-12-05T13:27:48,299 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:48,299 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:48,299 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. after waiting 0 ms 2024-12-05T13:27:48,299 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:48,300 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733405266798.4136a7e4b93de76fd0abc63fa3157b7b. 2024-12-05T13:27:48,301 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4136a7e4b93de76fd0abc63fa3157b7b: Waiting for close lock at 1733405268299Disabling compacts and flushes for region at 1733405268299Disabling writes for close at 1733405268299Writing region close event to WAL at 1733405268300 (+1 ms)Closed at 1733405268300 2024-12-05T13:27:48,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741870_1047 (size=93) 2024-12-05T13:27:48,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741870_1047 (size=93) 2024-12-05T13:27:48,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741870_1047 (size=93) 2024-12-05T13:27:48,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:48,308 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733405267608) 2024-12-05T13:27:48,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 400) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:49332 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:36118 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:32910 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 863) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=4422 (was 4666) 2024-12-05T13:27:48,335 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=4421 2024-12-05T13:27:48,351 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:48,357 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:48,358 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733405268358 2024-12-05T13:27:48,367 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358 2024-12-05T13:27:48,370 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:48,374 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cded77bc3abcea3c26750a024a987fbe, NAME => 'testSequentialEditLogSeqNum,,1733405268351.cded77bc3abcea3c26750a024a987fbe.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:48,374 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733405268351.cded77bc3abcea3c26750a024a987fbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:48,374 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,374 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,375 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe doesn't exist for region: cded77bc3abcea3c26750a024a987fbe on table testSequentialEditLogSeqNum 2024-12-05T13:27:48,376 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: cded77bc3abcea3c26750a024a987fbe on table testSequentialEditLogSeqNum 2024-12-05T13:27:48,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741875_1052 (size=62) 2024-12-05T13:27:48,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741875_1052 (size=62) 2024-12-05T13:27:48,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741875_1052 (size=62) 2024-12-05T13:27:48,389 INFO [StoreOpener-cded77bc3abcea3c26750a024a987fbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,391 INFO [StoreOpener-cded77bc3abcea3c26750a024a987fbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cded77bc3abcea3c26750a024a987fbe columnFamilyName a 2024-12-05T13:27:48,391 DEBUG [StoreOpener-cded77bc3abcea3c26750a024a987fbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:48,391 INFO [StoreOpener-cded77bc3abcea3c26750a024a987fbe-1 {}] regionserver.HStore(327): Store=cded77bc3abcea3c26750a024a987fbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:48,392 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,392 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,393 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,393 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,395 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cded77bc3abcea3c26750a024a987fbe 2024-12-05T13:27:48,398 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:48,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cded77bc3abcea3c26750a024a987fbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67653379, jitterRate=0.008113905787467957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:27:48,399 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cded77bc3abcea3c26750a024a987fbe: Writing region info on filesystem at 1733405268374Initializing all the Stores at 1733405268388 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405268388Cleaning up temporary data from old regions at 1733405268393 (+5 ms)Region opened successfully at 1733405268399 (+6 ms) 2024-12-05T13:27:48,414 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cded77bc3abcea3c26750a024a987fbe 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-05T13:27:48,441 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/.tmp/a/8c0b9281bc4343a5b4b670a66aeeed1a is 81, key is testSequentialEditLogSeqNum/a:x0/1733405268399/Put/seqid=0 2024-12-05T13:27:48,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741876_1053 (size=5833) 2024-12-05T13:27:48,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741876_1053 (size=5833) 2024-12-05T13:27:48,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741876_1053 (size=5833) 2024-12-05T13:27:48,453 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/.tmp/a/8c0b9281bc4343a5b4b670a66aeeed1a 2024-12-05T13:27:48,462 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/.tmp/a/8c0b9281bc4343a5b4b670a66aeeed1a as hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/a/8c0b9281bc4343a5b4b670a66aeeed1a 2024-12-05T13:27:48,472 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/a/8c0b9281bc4343a5b4b670a66aeeed1a, entries=10, sequenceid=13, filesize=5.7 K 2024-12-05T13:27:48,474 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for cded77bc3abcea3c26750a024a987fbe in 60ms, sequenceid=13, compaction requested=false 2024-12-05T13:27:48,474 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cded77bc3abcea3c26750a024a987fbe: 2024-12-05T13:27:48,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-05T13:27:48,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-05T13:27:48,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-05T13:27:48,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-05T13:27:48,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-05T13:27:48,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741874_1051 (size=3017) 2024-12-05T13:27:48,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741874_1051 (size=3017) 2024-12-05T13:27:48,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741874_1051 (size=3017) 2024-12-05T13:27:48,510 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358, size=2.9 K (3017bytes) 2024-12-05T13:27:48,510 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358 2024-12-05T13:27:48,511 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358 after 0ms 2024-12-05T13:27:48,514 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358 took 4ms 2024-12-05T13:27:48,518 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358 so closing down 2024-12-05T13:27:48,518 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:48,519 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733405268358.temp 2024-12-05T13:27:48,521 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000003-wal.1733405268358.temp 2024-12-05T13:27:48,521 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:48,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741877_1054 (size=2433) 2024-12-05T13:27:48,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741877_1054 (size=2433) 2024-12-05T13:27:48,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741877_1054 (size=2433) 2024-12-05T13:27:48,534 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000003-wal.1733405268358.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:48,536 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000003-wal.1733405268358.temp to hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000020 2024-12-05T13:27:48,536 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-12-05T13:27:48,536 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358, journal: Splitting hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358, size=2.9 K (3017bytes) at 1733405268510Finishing writing output for hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358 so closing down at 1733405268518 (+8 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000003-wal.1733405268358.temp at 1733405268521 (+3 ms)3 split writer threads finished at 1733405268521Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000003-wal.1733405268358.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733405268534 (+13 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000003-wal.1733405268358.temp to hdfs://localhost:44203/hbase/data/default/testSequentialEditLogSeqNum/cded77bc3abcea3c26750a024a987fbe/recovered.edits/0000000000000000020 at 1733405268536 (+2 ms)Processed 17 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:44203/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733405268350/wal.1733405268358, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1733405268536 2024-12-05T13:27:48,551 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:36118 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:32910 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=4396 (was 4421) 2024-12-05T13:27:48,568 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=4394 2024-12-05T13:27:48,584 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:48,586 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:48,619 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:48,625 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77370733, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-77370733, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:48,639 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77370733/hregion-77370733.1733405268626, exclude list is [], retry=0 2024-12-05T13:27:48,642 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:48,642 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:48,643 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:48,646 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77370733/hregion-77370733.1733405268626 2024-12-05T13:27:48,647 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:27:48,647 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8f41c7df1bbfea62f6a9d78425c37546, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:27:48,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741879_1056 (size=70) 2024-12-05T13:27:48,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741879_1056 (size=70) 2024-12-05T13:27:48,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741879_1056 (size=70) 2024-12-05T13:27:48,658 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:48,660 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,662 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName a 2024-12-05T13:27:48,662 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:48,663 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:48,664 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,666 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName b 2024-12-05T13:27:48,667 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:48,667 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:48,667 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,670 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName c 2024-12-05T13:27:48,670 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:48,671 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:48,671 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,673 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,673 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,675 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,675 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,676 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:48,677 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,680 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:48,681 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f41c7df1bbfea62f6a9d78425c37546; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61426847, jitterRate=-0.08466865122318268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:48,683 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f41c7df1bbfea62f6a9d78425c37546: Writing region info on filesystem at 1733405268658Initializing all the Stores at 1733405268659 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405268659Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405268660 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405268660Cleaning up temporary data from old regions at 1733405268675 (+15 ms)Region opened successfully at 1733405268682 (+7 ms) 2024-12-05T13:27:48,683 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f41c7df1bbfea62f6a9d78425c37546, disabling compactions & flushes 2024-12-05T13:27:48,683 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:48,683 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:48,683 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. after waiting 0 ms 2024-12-05T13:27:48,683 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:48,684 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:48,684 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f41c7df1bbfea62f6a9d78425c37546: Waiting for close lock at 1733405268683Disabling compacts and flushes for region at 1733405268683Disabling writes for close at 1733405268683Writing region close event to WAL at 1733405268684 (+1 ms)Closed at 1733405268684 2024-12-05T13:27:48,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741878_1055 (size=93) 2024-12-05T13:27:48,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741878_1055 (size=93) 2024-12-05T13:27:48,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741878_1055 (size=93) 2024-12-05T13:27:48,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:48,691 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77370733:(num 1733405268626) 2024-12-05T13:27:48,692 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:48,694 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:48,745 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695, exclude list is [], retry=0 2024-12-05T13:27:48,750 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:48,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:48,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:48,765 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 2024-12-05T13:27:48,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:48,765 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f41c7df1bbfea62f6a9d78425c37546, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:48,766 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:48,766 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,766 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,772 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,774 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName a 2024-12-05T13:27:48,774 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:48,775 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:48,775 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,776 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName b 2024-12-05T13:27:48,776 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:48,777 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:48,777 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,778 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName c 2024-12-05T13:27:48,778 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:48,779 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:48,779 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,780 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,781 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,782 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,782 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,783 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:48,784 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:48,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f41c7df1bbfea62f6a9d78425c37546; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69625762, jitterRate=0.03750470280647278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:48,786 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f41c7df1bbfea62f6a9d78425c37546: Writing region info on filesystem at 1733405268766Initializing all the Stores at 1733405268767 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405268767Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405268772 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405268772Cleaning up temporary data from old regions at 1733405268782 (+10 ms)Region opened successfully at 1733405268786 (+4 ms) 2024-12-05T13:27:48,790 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733405268789/Put/seqid=0 2024-12-05T13:27:48,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741881_1058 (size=4826) 2024-12-05T13:27:48,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741881_1058 (size=4826) 2024-12-05T13:27:48,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741881_1058 (size=4826) 2024-12-05T13:27:48,800 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44203/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 8f41c7df1bbfea62f6a9d78425c37546/a 2024-12-05T13:27:48,810 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-05T13:27:48,810 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-05T13:27:48,810 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f41c7df1bbfea62f6a9d78425c37546: 2024-12-05T13:27:48,812 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/a/a67cf04e9e264428a013224e9225d8dc_SeqId_3_ 2024-12-05T13:27:48,813 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44203/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8f41c7df1bbfea62f6a9d78425c37546/a as hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/a/a67cf04e9e264428a013224e9225d8dc_SeqId_3_ - updating store file list. 2024-12-05T13:27:48,818 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a67cf04e9e264428a013224e9225d8dc_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-05T13:27:48,819 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/a/a67cf04e9e264428a013224e9225d8dc_SeqId_3_ into 8f41c7df1bbfea62f6a9d78425c37546/a 2024-12-05T13:27:48,819 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44203/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8f41c7df1bbfea62f6a9d78425c37546/a (new location: hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/a/a67cf04e9e264428a013224e9225d8dc_SeqId_3_) 2024-12-05T13:27:48,863 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695, size=0 (0bytes) 2024-12-05T13:27:48,863 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 might be still open, length is 0 2024-12-05T13:27:48,863 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 2024-12-05T13:27:48,864 WARN [IPC Server handler 2 on default port 44203 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-05T13:27:48,864 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 after 1ms 2024-12-05T13:27:50,479 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:49432 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49432 dst: /127.0.0.1:40037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:50,478 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:36216 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38677:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36216 dst: /127.0.0.1:38677 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38677 remote=/127.0.0.1:36216]. Total timeout mills is 60000, 58347 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:50,483 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33154 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33154 dst: /127.0.0.1:43089 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741880_1059 (size=504) 2024-12-05T13:27:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741880_1059 (size=504) 2024-12-05T13:27:52,865 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 after 4002ms 2024-12-05T13:27:52,869 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 took 4006ms 2024-12-05T13:27:52,871 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695; continuing. 2024-12-05T13:27:52,871 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 so closing down 2024-12-05T13:27:52,871 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:52,873 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733405268695.temp 2024-12-05T13:27:52,875 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005-wal.1733405268695.temp 2024-12-05T13:27:52,876 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:52,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741882_1060 (size=238) 2024-12-05T13:27:52,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741882_1060 (size=238) 2024-12-05T13:27:52,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741882_1060 (size=238) 2024-12-05T13:27:52,885 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005-wal.1733405268695.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:52,887 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005-wal.1733405268695.temp to hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005 2024-12-05T13:27:52,887 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695, size=0, length=0, corrupted=false, cancelled=false 2024-12-05T13:27:52,887 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695, journal: Splitting hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695, size=0 (0bytes) at 1733405268863Finishing writing output for hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 so closing down at 1733405272871 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005-wal.1733405268695.temp at 1733405272875 (+4 ms)3 split writer threads finished at 1733405272876 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005-wal.1733405268695.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733405272885 (+9 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005-wal.1733405268695.temp to hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005 at 1733405272887 (+2 ms)Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695, size=0, length=0, corrupted=false, cancelled=false at 1733405272887 2024-12-05T13:27:52,889 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405268695 2024-12-05T13:27:52,890 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005 2024-12-05T13:27:52,890 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:52,892 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:52,905 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405272892, exclude list is [], retry=0 2024-12-05T13:27:52,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:52,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:52,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:52,915 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405272892 2024-12-05T13:27:52,916 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:52,916 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f41c7df1bbfea62f6a9d78425c37546, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:52,916 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:52,916 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,916 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,918 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,919 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName a 2024-12-05T13:27:52,919 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:52,925 DEBUG [StoreFileOpener-8f41c7df1bbfea62f6a9d78425c37546-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a67cf04e9e264428a013224e9225d8dc_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-05T13:27:52,925 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/a/a67cf04e9e264428a013224e9225d8dc_SeqId_3_ 2024-12-05T13:27:52,925 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:52,925 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,927 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName b 2024-12-05T13:27:52,927 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:52,927 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:52,928 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,929 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f41c7df1bbfea62f6a9d78425c37546 columnFamilyName c 2024-12-05T13:27:52,929 DEBUG [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:52,929 INFO [StoreOpener-8f41c7df1bbfea62f6a9d78425c37546-1 {}] regionserver.HStore(327): Store=8f41c7df1bbfea62f6a9d78425c37546/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:52,929 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,930 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,932 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,933 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005 2024-12-05T13:27:52,936 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005 2024-12-05T13:27:52,936 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f41c7df1bbfea62f6a9d78425c37546 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-05T13:27:52,951 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/.tmp/a/f33378e8f7a74e219af2cd5514dd811c is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733405268825/Put/seqid=0 2024-12-05T13:27:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741884_1062 (size=5149) 2024-12-05T13:27:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741884_1062 (size=5149) 2024-12-05T13:27:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741884_1062 (size=5149) 2024-12-05T13:27:52,962 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/.tmp/a/f33378e8f7a74e219af2cd5514dd811c 2024-12-05T13:27:52,969 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/.tmp/a/f33378e8f7a74e219af2cd5514dd811c as hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/a/f33378e8f7a74e219af2cd5514dd811c 2024-12-05T13:27:52,975 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/a/f33378e8f7a74e219af2cd5514dd811c, entries=1, sequenceid=5, filesize=5.0 K 2024-12-05T13:27:52,975 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8f41c7df1bbfea62f6a9d78425c37546 in 39ms, sequenceid=5, compaction requested=false; wal=null 2024-12-05T13:27:52,976 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/0000000000000000005 2024-12-05T13:27:52,978 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,978 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,979 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:52,981 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f41c7df1bbfea62f6a9d78425c37546 2024-12-05T13:27:52,983 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8f41c7df1bbfea62f6a9d78425c37546/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-05T13:27:52,984 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f41c7df1bbfea62f6a9d78425c37546; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72292331, jitterRate=0.07723967730998993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:52,984 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f41c7df1bbfea62f6a9d78425c37546: Writing region info on filesystem at 1733405272916Initializing all the Stores at 1733405272917 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405272917Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405272918 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405272918Obtaining lock to block concurrent updates at 1733405272936 (+18 ms)Preparing flush snapshotting stores in 8f41c7df1bbfea62f6a9d78425c37546 at 1733405272936Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733405272936Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. at 1733405272936Flushing 8f41c7df1bbfea62f6a9d78425c37546/a: creating writer at 1733405272936Flushing 8f41c7df1bbfea62f6a9d78425c37546/a: appending metadata at 1733405272950 (+14 ms)Flushing 8f41c7df1bbfea62f6a9d78425c37546/a: closing flushed file at 1733405272950Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21138ba4: reopening flushed file at 1733405272968 (+18 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8f41c7df1bbfea62f6a9d78425c37546 in 39ms, sequenceid=5, compaction requested=false; wal=null at 1733405272975 (+7 ms)Cleaning up temporary data from old regions at 1733405272978 (+3 ms)Region opened successfully at 1733405272984 (+6 ms) 2024-12-05T13:27:52,989 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f41c7df1bbfea62f6a9d78425c37546, disabling compactions & flushes 2024-12-05T13:27:52,989 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:52,989 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:52,989 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. after waiting 0 ms 2024-12-05T13:27:52,989 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:52,990 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733405268585.8f41c7df1bbfea62f6a9d78425c37546. 2024-12-05T13:27:52,990 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f41c7df1bbfea62f6a9d78425c37546: Waiting for close lock at 1733405272989Disabling compacts and flushes for region at 1733405272989Disabling writes for close at 1733405272989Writing region close event to WAL at 1733405272990 (+1 ms)Closed at 1733405272990 2024-12-05T13:27:52,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741883_1061 (size=93) 2024-12-05T13:27:52,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741883_1061 (size=93) 2024-12-05T13:27:52,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741883_1061 (size=93) 2024-12-05T13:27:52,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:52,997 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733405272892) 2024-12-05T13:27:53,013 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 409) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44203 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1707291693_22 at /127.0.0.1:33184 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1707291693_22 at /127.0.0.1:36266 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1707291693_22 at /127.0.0.1:49454 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:44203 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=212 (was 222), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=4229 (was 4394) 2024-12-05T13:27:53,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=212, ProcessCount=12, AvailableMemoryMB=4227 2024-12-05T13:27:53,042 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:53,047 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T13:27:53,052 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 88aaa70743e7,43665,1733405256332 2024-12-05T13:27:53,054 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7febadd 2024-12-05T13:27:53,055 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T13:27:53,058 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T13:27:53,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T13:27:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-05T13:27:53,072 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T13:27:53,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-05T13:27:53,074 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:53,076 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T13:27:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:27:53,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741885_1063 (size=694) 2024-12-05T13:27:53,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741885_1063 (size=694) 2024-12-05T13:27:53,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741885_1063 (size=694) 2024-12-05T13:27:53,090 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2889c120f533ce016b2cbcb6b393a73, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c 2024-12-05T13:27:53,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741886_1064 (size=77) 2024-12-05T13:27:53,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741886_1064 (size=77) 2024-12-05T13:27:53,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741886_1064 (size=77) 2024-12-05T13:27:53,106 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:53,106 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing d2889c120f533ce016b2cbcb6b393a73, disabling compactions & flushes 2024-12-05T13:27:53,106 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,106 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,106 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. after waiting 0 ms 2024-12-05T13:27:53,107 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,107 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,107 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2889c120f533ce016b2cbcb6b393a73: Waiting for close lock at 1733405273106Disabling compacts and flushes for region at 1733405273106Disabling writes for close at 1733405273107 (+1 ms)Writing region close event to WAL at 1733405273107Closed at 1733405273107 2024-12-05T13:27:53,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T13:27:53,116 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733405273109"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733405273109"}]},"ts":"1733405273109"} 2024-12-05T13:27:53,120 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T13:27:53,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T13:27:53,125 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733405273122"}]},"ts":"1733405273122"} 2024-12-05T13:27:53,130 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-05T13:27:53,131 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:27:53,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:27:53,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:27:53,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-05T13:27:53,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:27:53,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:27:53,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T13:27:53,133 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:27:53,133 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:27:53,133 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T13:27:53,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:27:53,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN}] 2024-12-05T13:27:53,138 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN 2024-12-05T13:27:53,140 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN; state=OFFLINE, location=88aaa70743e7,35441,1733405257297; forceNewPlan=false, retain=false 2024-12-05T13:27:53,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:27:53,293 INFO [88aaa70743e7:43665 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T13:27:53,295 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPENING, regionLocation=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:53,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN because future has completed 2024-12-05T13:27:53,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297}] 2024-12-05T13:27:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:27:53,461 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,462 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d2889c120f533ce016b2cbcb6b393a73, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:53,462 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,462 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:53,463 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,463 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,465 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,467 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf1 2024-12-05T13:27:53,467 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:53,467 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:53,468 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,469 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf2 2024-12-05T13:27:53,469 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:53,470 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:53,470 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,471 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,472 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,472 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,473 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,474 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-05T13:27:53,475 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,478 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:53,478 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d2889c120f533ce016b2cbcb6b393a73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64462223, jitterRate=-0.039438024163246155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-05T13:27:53,478 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,479 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d2889c120f533ce016b2cbcb6b393a73: Running coprocessor pre-open hook at 1733405273463Writing region info on filesystem at 1733405273463Initializing all the Stores at 1733405273464 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405273464Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405273464Cleaning up temporary data from old regions at 1733405273473 (+9 ms)Running coprocessor post-open hooks at 1733405273478 (+5 ms)Region opened successfully at 1733405273479 (+1 ms) 2024-12-05T13:27:53,481 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., pid=6, masterSystemTime=1733405273453 2024-12-05T13:27:53,484 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,484 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:53,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297 because future has completed 2024-12-05T13:27:53,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T13:27:53,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297 in 190 msec 2024-12-05T13:27:53,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T13:27:53,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN in 358 msec 2024-12-05T13:27:53,498 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T13:27:53,498 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733405273498"}]},"ts":"1733405273498"} 2024-12-05T13:27:53,501 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-05T13:27:53,502 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T13:27:53,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 438 msec 2024-12-05T13:27:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:27:53,712 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-05T13:27:53,712 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-05T13:27:53,713 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:27:53,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-05T13:27:53,721 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:27:53,722 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-05T13:27:53,738 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=2] 2024-12-05T13:27:53,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=d2889c120f533ce016b2cbcb6b393a73, source=88aaa70743e7,35441,1733405257297, destination=88aaa70743e7,33089,1733405257168, warming up region on 88aaa70743e7,33089,1733405257168 2024-12-05T13:27:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:27:53,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=d2889c120f533ce016b2cbcb6b393a73, source=88aaa70743e7,35441,1733405257297, destination=88aaa70743e7,33089,1733405257168, running balancer 2024-12-05T13:27:53,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56371, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:27:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE 2024-12-05T13:27:53,761 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE 2024-12-05T13:27:53,764 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=CLOSING, regionLocation=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:53,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE because future has completed 2024-12-05T13:27:53,769 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T13:27:53,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297}] 2024-12-05T13:27:53,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(7855): Warmup {ENCODED => d2889c120f533ce016b2cbcb6b393a73, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:53,773 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,775 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf1 2024-12-05T13:27:53,775 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:53,776 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:53,776 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,777 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf2 2024-12-05T13:27:53,777 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:53,778 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(1722): Closing d2889c120f533ce016b2cbcb6b393a73, disabling compactions & flushes 2024-12-05T13:27:53,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. after waiting 0 ms 2024-12-05T13:27:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] regionserver.HRegion(1676): Region close journal for d2889c120f533ce016b2cbcb6b393a73: Waiting for close lock at 1733405273778Disabling compacts and flushes for region at 1733405273778Disabling writes for close at 1733405273779 (+1 ms)Writing region close event to WAL at 1733405273781 (+2 ms)Closed at 1733405273781 2024-12-05T13:27:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-05T13:27:53,928 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,928 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T13:27:53,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing d2889c120f533ce016b2cbcb6b393a73, disabling compactions & flushes 2024-12-05T13:27:53,929 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. after waiting 0 ms 2024-12-05T13:27:53,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,929 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing d2889c120f533ce016b2cbcb6b393a73 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-05T13:27:53,950 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 is 35, key is r1/cf1:q/1733405273741/Put/seqid=0 2024-12-05T13:27:53,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741887_1065 (size=4783) 2024-12-05T13:27:53,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741887_1065 (size=4783) 2024-12-05T13:27:53,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741887_1065 (size=4783) 2024-12-05T13:27:53,960 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 2024-12-05T13:27:53,971 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 2024-12-05T13:27:53,980 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2, entries=1, sequenceid=5, filesize=4.7 K 2024-12-05T13:27:53,981 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for d2889c120f533ce016b2cbcb6b393a73 in 52ms, sequenceid=5, compaction requested=false 2024-12-05T13:27:53,981 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-05T13:27:53,987 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-05T13:27:53,990 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:53,990 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for d2889c120f533ce016b2cbcb6b393a73: Waiting for close lock at 1733405273929Running coprocessor pre-close hooks at 1733405273929Disabling compacts and flushes for region at 1733405273929Disabling writes for close at 1733405273929Obtaining lock to block concurrent updates at 1733405273929Preparing flush snapshotting stores in d2889c120f533ce016b2cbcb6b393a73 at 1733405273929Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733405273930 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. at 1733405273931 (+1 ms)Flushing d2889c120f533ce016b2cbcb6b393a73/cf1: creating writer at 1733405273931Flushing d2889c120f533ce016b2cbcb6b393a73/cf1: appending metadata at 1733405273949 (+18 ms)Flushing d2889c120f533ce016b2cbcb6b393a73/cf1: closing flushed file at 1733405273950 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4419f3e4: reopening flushed file at 1733405273967 (+17 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for d2889c120f533ce016b2cbcb6b393a73 in 52ms, sequenceid=5, compaction requested=false at 1733405273981 (+14 ms)Writing region close event to WAL at 1733405273982 (+1 ms)Running coprocessor post-close hooks at 1733405273988 (+6 ms)Closed at 1733405273990 (+2 ms) 2024-12-05T13:27:53,990 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding d2889c120f533ce016b2cbcb6b393a73 move to 88aaa70743e7,33089,1733405257168 record at close sequenceid=5 2024-12-05T13:27:53,994 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:53,995 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=CLOSED 2024-12-05T13:27:54,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297 because future has completed 2024-12-05T13:27:54,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T13:27:54,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297 in 235 msec 2024-12-05T13:27:54,010 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE; state=CLOSED, location=88aaa70743e7,33089,1733405257168; forceNewPlan=false, retain=false 2024-12-05T13:27:54,160 INFO [88aaa70743e7:43665 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T13:27:54,161 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPENING, regionLocation=88aaa70743e7,33089,1733405257168 2024-12-05T13:27:54,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE because future has completed 2024-12-05T13:27:54,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,33089,1733405257168}] 2024-12-05T13:27:54,323 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,323 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => d2889c120f533ce016b2cbcb6b393a73, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:54,323 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,323 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:54,324 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,324 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,326 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,327 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf1 2024-12-05T13:27:54,327 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:54,335 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 2024-12-05T13:27:54,335 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:54,336 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,337 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf2 2024-12-05T13:27:54,337 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:54,338 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:54,338 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,339 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,341 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,341 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,341 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,342 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-05T13:27:54,344 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,345 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened d2889c120f533ce016b2cbcb6b393a73; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61290595, jitterRate=-0.08669896423816681}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-05T13:27:54,345 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,346 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for d2889c120f533ce016b2cbcb6b393a73: Running coprocessor pre-open hook at 1733405274324Writing region info on filesystem at 1733405274324Initializing all the Stores at 1733405274325 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405274325Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405274326 (+1 ms)Cleaning up temporary data from old regions at 1733405274341 (+15 ms)Running coprocessor post-open hooks at 1733405274345 (+4 ms)Region opened successfully at 1733405274346 (+1 ms) 2024-12-05T13:27:54,348 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., pid=9, masterSystemTime=1733405274318 2024-12-05T13:27:54,350 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,351 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,351 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPEN, openSeqNum=9, regionLocation=88aaa70743e7,33089,1733405257168 2024-12-05T13:27:54,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,33089,1733405257168 because future has completed 2024-12-05T13:27:54,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-05T13:27:54,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,33089,1733405257168 in 193 msec 2024-12-05T13:27:54,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE in 601 msec 2024-12-05T13:27:54,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:27:54,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:27:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:49966 deadline: 1733405334428, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=33089 startCode=1733405257168. As of locationSeqNum=5. 2024-12-05T13:27:54,453 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=33089 startCode=1733405257168. As of locationSeqNum=5. 2024-12-05T13:27:54,454 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=33089 startCode=1733405257168. As of locationSeqNum=5. 2024-12-05T13:27:54,454 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,33089,1733405257168, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=33089 startCode=1733405257168. As of locationSeqNum=5. 2024-12-05T13:27:54,560 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:27:54,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:27:54,574 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d2889c120f533ce016b2cbcb6b393a73 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-05T13:27:54,591 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf1/555b6c29d58c4b1eaab78c477342dd3f is 29, key is r1/cf1:/1733405274564/DeleteFamily/seqid=0 2024-12-05T13:27:54,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741888_1066 (size=4906) 2024-12-05T13:27:54,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741888_1066 (size=4906) 2024-12-05T13:27:54,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741888_1066 (size=4906) 2024-12-05T13:27:54,599 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf1/555b6c29d58c4b1eaab78c477342dd3f 2024-12-05T13:27:54,605 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 555b6c29d58c4b1eaab78c477342dd3f 2024-12-05T13:27:54,620 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf2/c4e0c0eb40c5460faba770bbecdf3b87 is 29, key is r1/cf2:/1733405274564/DeleteFamily/seqid=0 2024-12-05T13:27:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741889_1067 (size=4906) 2024-12-05T13:27:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741889_1067 (size=4906) 2024-12-05T13:27:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741889_1067 (size=4906) 2024-12-05T13:27:54,627 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf2/c4e0c0eb40c5460faba770bbecdf3b87 2024-12-05T13:27:54,633 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c4e0c0eb40c5460faba770bbecdf3b87 2024-12-05T13:27:54,634 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf1/555b6c29d58c4b1eaab78c477342dd3f as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/555b6c29d58c4b1eaab78c477342dd3f 2024-12-05T13:27:54,641 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 555b6c29d58c4b1eaab78c477342dd3f 2024-12-05T13:27:54,641 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/555b6c29d58c4b1eaab78c477342dd3f, entries=1, sequenceid=12, filesize=4.8 K 2024-12-05T13:27:54,643 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf2/c4e0c0eb40c5460faba770bbecdf3b87 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/c4e0c0eb40c5460faba770bbecdf3b87 2024-12-05T13:27:54,650 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c4e0c0eb40c5460faba770bbecdf3b87 2024-12-05T13:27:54,650 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/c4e0c0eb40c5460faba770bbecdf3b87, entries=1, sequenceid=12, filesize=4.8 K 2024-12-05T13:27:54,651 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for d2889c120f533ce016b2cbcb6b393a73 in 77ms, sequenceid=12, compaction requested=false 2024-12-05T13:27:54,651 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d2889c120f533ce016b2cbcb6b393a73: 2024-12-05T13:27:54,654 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T13:27:54,657 DEBUG [Time-limited test {}] regionserver.HStore(1541): d2889c120f533ce016b2cbcb6b393a73/cf1 is initiating major compaction (all files) 2024-12-05T13:27:54,657 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:27:54,657 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:54,658 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of d2889c120f533ce016b2cbcb6b393a73/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,658 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2, hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/555b6c29d58c4b1eaab78c477342dd3f] into tmpdir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp, totalSize=9.5 K 2024-12-05T13:27:54,660 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 059f1926b3bd4b5da6b8f256f47bcdb2, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733405273741 2024-12-05T13:27:54,661 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 555b6c29d58c4b1eaab78c477342dd3f, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-05T13:27:54,683 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): d2889c120f533ce016b2cbcb6b393a73#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T13:27:54,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741890_1068 (size=4626) 2024-12-05T13:27:54,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741890_1068 (size=4626) 2024-12-05T13:27:54,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741890_1068 (size=4626) 2024-12-05T13:27:54,702 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf1/2d17b15d7d59457cb38a48ed713a8a89 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/2d17b15d7d59457cb38a48ed713a8a89 2024-12-05T13:27:54,720 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in d2889c120f533ce016b2cbcb6b393a73/cf1 of d2889c120f533ce016b2cbcb6b393a73 into 2d17b15d7d59457cb38a48ed713a8a89(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T13:27:54,720 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for d2889c120f533ce016b2cbcb6b393a73: 2024-12-05T13:27:54,720 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-05T13:27:54,720 DEBUG [Time-limited test {}] regionserver.HStore(1541): d2889c120f533ce016b2cbcb6b393a73/cf2 is initiating major compaction (all files) 2024-12-05T13:27:54,720 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:27:54,720 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:27:54,720 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of d2889c120f533ce016b2cbcb6b393a73/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,721 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/c4e0c0eb40c5460faba770bbecdf3b87] into tmpdir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp, totalSize=4.8 K 2024-12-05T13:27:54,721 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c4e0c0eb40c5460faba770bbecdf3b87, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-05T13:27:54,728 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): d2889c120f533ce016b2cbcb6b393a73#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T13:27:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741891_1069 (size=4592) 2024-12-05T13:27:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741891_1069 (size=4592) 2024-12-05T13:27:54,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741891_1069 (size=4592) 2024-12-05T13:27:54,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/.tmp/cf2/bc142118035948c1bd0502d9e7d9d1b3 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/bc142118035948c1bd0502d9e7d9d1b3 2024-12-05T13:27:54,758 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in d2889c120f533ce016b2cbcb6b393a73/cf2 of d2889c120f533ce016b2cbcb6b393a73 into bc142118035948c1bd0502d9e7d9d1b3(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T13:27:54,759 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for d2889c120f533ce016b2cbcb6b393a73: 2024-12-05T13:27:54,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=d2889c120f533ce016b2cbcb6b393a73, source=88aaa70743e7,33089,1733405257168, destination=88aaa70743e7,35441,1733405257297, warming up region on 88aaa70743e7,35441,1733405257297 2024-12-05T13:27:54,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=d2889c120f533ce016b2cbcb6b393a73, source=88aaa70743e7,33089,1733405257168, destination=88aaa70743e7,35441,1733405257297, running balancer 2024-12-05T13:27:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE 2024-12-05T13:27:54,766 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE 2024-12-05T13:27:54,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(7855): Warmup {ENCODED => d2889c120f533ce016b2cbcb6b393a73, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:54,768 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=CLOSING, regionLocation=88aaa70743e7,33089,1733405257168 2024-12-05T13:27:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:54,769 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,770 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf1 2024-12-05T13:27:54,770 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:54,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE because future has completed 2024-12-05T13:27:54,772 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T13:27:54,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,33089,1733405257168}] 2024-12-05T13:27:54,779 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 2024-12-05T13:27:54,785 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/2d17b15d7d59457cb38a48ed713a8a89 2024-12-05T13:27:54,791 INFO [StoreFileOpener-d2889c120f533ce016b2cbcb6b393a73-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 555b6c29d58c4b1eaab78c477342dd3f 2024-12-05T13:27:54,792 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/555b6c29d58c4b1eaab78c477342dd3f 2024-12-05T13:27:54,792 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:54,792 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,794 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf2 2024-12-05T13:27:54,794 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:54,803 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/bc142118035948c1bd0502d9e7d9d1b3 2024-12-05T13:27:54,810 INFO [StoreFileOpener-d2889c120f533ce016b2cbcb6b393a73-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c4e0c0eb40c5460faba770bbecdf3b87 2024-12-05T13:27:54,810 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/c4e0c0eb40c5460faba770bbecdf3b87 2024-12-05T13:27:54,810 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(1722): Closing d2889c120f533ce016b2cbcb6b393a73, disabling compactions & flushes 2024-12-05T13:27:54,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. after waiting 0 ms 2024-12-05T13:27:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-05T13:27:54,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35441 {}] regionserver.HRegion(1676): Region close journal for d2889c120f533ce016b2cbcb6b393a73: Waiting for close lock at 1733405274810Disabling compacts and flushes for region at 1733405274810Disabling writes for close at 1733405274811 (+1 ms)Writing region close event to WAL at 1733405274816 (+5 ms)Closed at 1733405274816 2024-12-05T13:27:54,927 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,927 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T13:27:54,927 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing d2889c120f533ce016b2cbcb6b393a73, disabling compactions & flushes 2024-12-05T13:27:54,927 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,927 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,927 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. after waiting 0 ms 2024-12-05T13:27:54,927 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,928 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2, hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/555b6c29d58c4b1eaab78c477342dd3f] to archive 2024-12-05T13:27:54,931 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T13:27:54,936 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/059f1926b3bd4b5da6b8f256f47bcdb2 2024-12-05T13:27:54,938 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/555b6c29d58c4b1eaab78c477342dd3f to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/555b6c29d58c4b1eaab78c477342dd3f 2024-12-05T13:27:54,952 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/c4e0c0eb40c5460faba770bbecdf3b87] to archive 2024-12-05T13:27:54,953 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T13:27:54,955 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/c4e0c0eb40c5460faba770bbecdf3b87 to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/c4e0c0eb40c5460faba770bbecdf3b87 2024-12-05T13:27:54,962 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-05T13:27:54,963 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:54,963 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for d2889c120f533ce016b2cbcb6b393a73: Waiting for close lock at 1733405274927Running coprocessor pre-close hooks at 1733405274927Disabling compacts and flushes for region at 1733405274927Disabling writes for close at 1733405274927Writing region close event to WAL at 1733405274958 (+31 ms)Running coprocessor post-close hooks at 1733405274963 (+5 ms)Closed at 1733405274963 2024-12-05T13:27:54,963 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding d2889c120f533ce016b2cbcb6b393a73 move to 88aaa70743e7,35441,1733405257297 record at close sequenceid=12 2024-12-05T13:27:54,967 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:54,968 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=CLOSED 2024-12-05T13:27:54,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,33089,1733405257168 because future has completed 2024-12-05T13:27:54,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T13:27:54,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,33089,1733405257168 in 202 msec 2024-12-05T13:27:54,978 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE; state=CLOSED, location=88aaa70743e7,35441,1733405257297; forceNewPlan=false, retain=false 2024-12-05T13:27:55,129 INFO [88aaa70743e7:43665 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T13:27:55,129 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPENING, regionLocation=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:55,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE because future has completed 2024-12-05T13:27:55,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297}] 2024-12-05T13:27:55,291 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,291 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => d2889c120f533ce016b2cbcb6b393a73, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:55,291 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,291 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:55,292 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,292 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,293 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,295 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf1 2024-12-05T13:27:55,295 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:55,306 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/2d17b15d7d59457cb38a48ed713a8a89 2024-12-05T13:27:55,306 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:55,306 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,307 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf2 2024-12-05T13:27:55,307 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:55,314 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/bc142118035948c1bd0502d9e7d9d1b3 2024-12-05T13:27:55,314 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:55,314 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,315 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,317 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,318 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,318 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,319 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-05T13:27:55,320 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,321 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened d2889c120f533ce016b2cbcb6b393a73; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60816594, jitterRate=-0.09376212954521179}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-05T13:27:55,322 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,322 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for d2889c120f533ce016b2cbcb6b393a73: Running coprocessor pre-open hook at 1733405275292Writing region info on filesystem at 1733405275292Initializing all the Stores at 1733405275293 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405275293Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405275293Cleaning up temporary data from old regions at 1733405275318 (+25 ms)Running coprocessor post-open hooks at 1733405275322 (+4 ms)Region opened successfully at 1733405275322 2024-12-05T13:27:55,324 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., pid=12, masterSystemTime=1733405275285 2024-12-05T13:27:55,328 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,328 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,329 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPEN, openSeqNum=18, regionLocation=88aaa70743e7,35441,1733405257297 2024-12-05T13:27:55,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297 because future has completed 2024-12-05T13:27:55,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-05T13:27:55,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,35441,1733405257297 in 199 msec 2024-12-05T13:27:55,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, REOPEN/MOVE in 571 msec 2024-12-05T13:27:55,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:27:55,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:27:55,372 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 88aaa70743e7,35441,1733405257297: testing ***** 2024-12-05T13:27:55,372 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-05T13:27:55,375 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-05T13:27:55,377 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-05T13:27:55,384 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-05T13:27:55,385 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-05T13:27:55,397 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 318765776 }, "NonHeapMemoryUsage": { "committed": 170786816, "init": 7667712, "max": -1, "used": 168216576 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 52, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 20, "ProcessCallTime_98th_percentile": 52, "ProcessCallTime_99th_percentile": 52, "ProcessCallTime_99.9th_percentile": 52, "ProcessCallTime_TimeRangeCount_0-1": 10, "ProcessCallTime_TimeRangeCount_1-3": 14, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 52, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 7, "TotalCallTime_95th_percentile": 20, "TotalCallTime_98th_percentile": 52, "TotalCallTime_99th_percentile": 52, "TotalCallTime_99.9th_percentile": 52, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 15, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5773 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:61512", "tag.serverName": "88aaa70743e7,44873,1733405257343", "tag.clusterId": "3e1a98ee-101f-4ab3-b69a-238955073761", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733405257343, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-05T13:27:55,400 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43665 {}] master.MasterRpcServices(700): 88aaa70743e7,35441,1733405257297 reported a fatal error: ***** ABORTING region server 88aaa70743e7,35441,1733405257297: testing ***** 2024-12-05T13:27:55,403 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88aaa70743e7,35441,1733405257297' ***** 2024-12-05T13:27:55,403 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-05T13:27:55,403 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T13:27:55,404 INFO [RS:1;88aaa70743e7:35441 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-05T13:27:55,404 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T13:27:55,404 INFO [RS:1;88aaa70743e7:35441 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-05T13:27:55,404 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(3091): Received CLOSE for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33089 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:56784 deadline: 1733405335404, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=35441 startCode=1733405257297. As of locationSeqNum=12. 2024-12-05T13:27:55,405 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(956): aborting server 88aaa70743e7,35441,1733405257297 2024-12-05T13:27:55,405 INFO [RS:1;88aaa70743e7:35441 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:27:55,405 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,33089,1733405257168, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,33089,1733405257168, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=35441 startCode=1733405257297. As of locationSeqNum=12. 2024-12-05T13:27:55,405 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,33089,1733405257168, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=35441 startCode=1733405257297. As of locationSeqNum=12. 2024-12-05T13:27:55,405 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,33089,1733405257168, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=88aaa70743e7 port=35441 startCode=1733405257297. As of locationSeqNum=12. 2024-12-05T13:27:55,405 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2889c120f533ce016b2cbcb6b393a73, disabling compactions & flushes 2024-12-05T13:27:55,405 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,405 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,405 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. after waiting 0 ms 2024-12-05T13:27:55,405 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,406 INFO [RS:1;88aaa70743e7:35441 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;88aaa70743e7:35441. 2024-12-05T13:27:55,406 DEBUG [RS:1;88aaa70743e7:35441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:27:55,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 88aaa70743e7,35441,1733405257297 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:55,408 DEBUG [RS:1;88aaa70743e7:35441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:27:55,409 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T13:27:55,409 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T13:27:55,409 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T13:27:55,409 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 88aaa70743e7,35441,1733405257297 aborting 2024-12-05T13:27:55,409 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T13:27:55,409 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 88aaa70743e7,35441,1733405257297 aborting 2024-12-05T13:27:55,409 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 from cache 2024-12-05T13:27:55,415 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T13:27:55,415 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T13:27:55,415 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d2889c120f533ce016b2cbcb6b393a73=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.} 2024-12-05T13:27:55,415 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T13:27:55,415 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T13:27:55,415 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T13:27:55,415 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T13:27:55,415 DEBUG [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:55,416 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,416 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2889c120f533ce016b2cbcb6b393a73: Waiting for close lock at 1733405275405Running coprocessor pre-close hooks at 1733405275405Disabling compacts and flushes for region at 1733405275405Disabling writes for close at 1733405275405Writing region close event to WAL at 1733405275416 (+11 ms)Running coprocessor post-close hooks at 1733405275416Closed at 1733405275416 2024-12-05T13:27:55,416 ERROR [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-05T13:27:55,416 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T13:27:55,416 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:55,417 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T13:27:55,417 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733405275415Running coprocessor pre-close hooks at 1733405275415Disabling compacts and flushes for region at 1733405275415Disabling writes for close at 1733405275415Writing region close event to WAL at 1733405275416 (+1 ms)Running coprocessor post-close hooks at 1733405275416Closed at 1733405275416 2024-12-05T13:27:55,417 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T13:27:55,433 INFO [regionserver/88aaa70743e7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:27:55,520 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:27:55,522 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1] 2024-12-05T13:27:55,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 88aaa70743e7,35441,1733405257297 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:55,523 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 88aaa70743e7,35441,1733405257297 aborting 2024-12-05T13:27:55,523 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 88aaa70743e7,35441,1733405257297 aborting 2024-12-05T13:27:55,524 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 from cache 2024-12-05T13:27:55,616 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(976): stopping server 88aaa70743e7,35441,1733405257297; all regions closed. 2024-12-05T13:27:55,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741836_1012 (size=6845) 2024-12-05T13:27:55,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741836_1012 (size=6845) 2024-12-05T13:27:55,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741836_1012 (size=6845) 2024-12-05T13:27:55,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741834_1010 (size=2142) 2024-12-05T13:27:55,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741834_1010 (size=2142) 2024-12-05T13:27:55,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741834_1010 (size=2142) 2024-12-05T13:27:55,628 DEBUG [RS:1;88aaa70743e7:35441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:27:55,629 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:27:55,629 INFO [RS:1;88aaa70743e7:35441 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T13:27:55,629 INFO [RS:1;88aaa70743e7:35441 {}] hbase.ChoreService(370): Chore service for: regionserver/88aaa70743e7:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T13:27:55,629 INFO [RS:1;88aaa70743e7:35441 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T13:27:55,629 INFO [regionserver/88aaa70743e7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T13:27:55,629 INFO [RS:1;88aaa70743e7:35441 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35441 2024-12-05T13:27:55,696 INFO [RS:1;88aaa70743e7:35441 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:27:55,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T13:27:55,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88aaa70743e7,35441,1733405257297 2024-12-05T13:27:55,696 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007feebc8e1888@7924f04d rejected from java.util.concurrent.ThreadPoolExecutor@3c26710d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 12] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T13:27:55,704 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88aaa70743e7,35441,1733405257297] 2024-12-05T13:27:55,711 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88aaa70743e7,35441,1733405257297 already deleted, retry=false 2024-12-05T13:27:55,712 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 88aaa70743e7,35441,1733405257297 on 88aaa70743e7,43665,1733405256332 2024-12-05T13:27:55,718 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 88aaa70743e7,35441,1733405257297, splitWal=true, meta=true 2024-12-05T13:27:55,721 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 88aaa70743e7,35441,1733405257297 (carryingMeta=true) 88aaa70743e7,35441,1733405257297/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@200d43e4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-05T13:27:55,721 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 88aaa70743e7,35441,1733405257297, splitWal=true, meta=true 2024-12-05T13:27:55,723 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 88aaa70743e7,35441,1733405257297, splitWal=true, meta=true, isMeta: true 2024-12-05T13:27:55,726 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting 2024-12-05T13:27:55,727 INFO [PEWorker-1 {}] master.SplitWALManager(105): 88aaa70743e7,35441,1733405257297 WAL count=1, meta=true 2024-12-05T13:27:55,729 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:27:55,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta}] 2024-12-05T13:27:55,731 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1] 2024-12-05T13:27:55,733 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 88aaa70743e7:35441 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 88aaa70743e7/172.17.0.3:35441 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:55,734 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:35441 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 88aaa70743e7/172.17.0.3:35441 2024-12-05T13:27:55,734 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-05T13:27:55,734 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 from cache 2024-12-05T13:27:55,734 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 88aaa70743e7:35441 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 88aaa70743e7/172.17.0.3:35441 2024-12-05T13:27:55,736 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:55,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, worker=88aaa70743e7,44873,1733405257343}] 2024-12-05T13:27:55,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:27:55,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35441-0x101a6daed470002, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:27:55,804 INFO [RS:1;88aaa70743e7:35441 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:27:55,805 INFO [RS:1;88aaa70743e7:35441 {}] regionserver.HRegionServer(1031): Exiting; stopping=88aaa70743e7,35441,1733405257297; zookeeper connection closed. 2024-12-05T13:27:55,805 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@538f6d55 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@538f6d55 2024-12-05T13:27:55,901 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:27:55,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39743, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:27:55,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-05T13:27:55,926 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, size=6.7 K (6845bytes) 2024-12-05T13:27:55,926 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta 2024-12-05T13:27:55,927 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta after 1ms 2024-12-05T13:27:55,929 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta took 3ms 2024-12-05T13:27:55,938 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-05T13:27:55,939 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta so closing down 2024-12-05T13:27:55,939 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:55,940 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta.temp 2024-12-05T13:27:55,942 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta.temp 2024-12-05T13:27:55,942 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:55,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741892_1070 (size=6604) 2024-12-05T13:27:55,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741892_1070 (size=6604) 2024-12-05T13:27:55,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741892_1070 (size=6604) 2024-12-05T13:27:55,954 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:55,955 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta.temp to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-05T13:27:55,955 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 24 ms; skipped=1; WAL=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-12-05T13:27:55,956 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, journal: Splitting hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, size=6.7 K (6845bytes) at 1733405275926Finishing writing output for hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta so closing down at 1733405275939 (+13 ms)Creating recovered edits writer path=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta.temp at 1733405275942 (+3 ms)3 split writer threads finished at 1733405275942Closed recovered edits writer path=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733405275954 (+12 ms)Rename recovered edits hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta.temp to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733405275955 (+1 ms)Processed 16 edits across 1 Regions in 24 ms; skipped=1; WAL=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1733405275956 (+1 ms) 2024-12-05T13:27:55,956 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta 2024-12-05T13:27:55,957 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-05T13:27:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-05T13:27:55,964 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs 2024-12-05T13:27:55,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-05T13:27:55,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, worker=88aaa70743e7,44873,1733405257343 in 226 msec 2024-12-05T13:27:55,969 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:55,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T13:27:55,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C35441%2C1733405257297.meta.1733405259280.meta, worker=88aaa70743e7,44873,1733405257343 in 240 msec 2024-12-05T13:27:55,975 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting dir is empty, no logs to split. 2024-12-05T13:27:55,975 INFO [PEWorker-1 {}] master.SplitWALManager(105): 88aaa70743e7,35441,1733405257297 WAL count=0, meta=true 2024-12-05T13:27:55,975 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 88aaa70743e7,35441,1733405257297 WAL splitting is done? wals=0, meta=true 2024-12-05T13:27:55,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T13:27:55,979 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:27:55,980 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:27:56,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:27:56,041 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1] 2024-12-05T13:27:56,041 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 88aaa70743e7:35441 this server is in the failed servers list 2024-12-05T13:27:56,042 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=88aaa70743e7:35441 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 88aaa70743e7:35441 2024-12-05T13:27:56,042 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 88aaa70743e7:35441 2024-12-05T13:27:56,042 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 from cache 2024-12-05T13:27:56,130 DEBUG [88aaa70743e7:43665 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T13:27:56,130 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:27:56,131 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:27:56,131 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:27:56,131 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:27:56,131 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:27:56,131 INFO [88aaa70743e7:43665 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:27:56,131 INFO [88aaa70743e7:43665 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:27:56,131 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:27:56,131 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:56,132 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,44873,1733405257343, state=OPENING 2024-12-05T13:27:56,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:56,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:56,142 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:27:56,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:56,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,44873,1733405257343}] 2024-12-05T13:27:56,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:56,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:56,142 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:56,301 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T13:27:56,301 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:56,302 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T13:27:56,304 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C44873%2C1733405257343.meta, suffix=.meta, logDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,44873,1733405257343, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs, maxLogs=32 2024-12-05T13:27:56,319 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,44873,1733405257343/88aaa70743e7%2C44873%2C1733405257343.meta.1733405276305.meta, exclude list is [], retry=0 2024-12-05T13:27:56,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:56,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:56,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:56,326 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,44873,1733405257343/88aaa70743e7%2C44873%2C1733405257343.meta.1733405276305.meta 2024-12-05T13:27:56,326 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:56,326 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:56,327 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T13:27:56,327 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T13:27:56,327 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T13:27:56,327 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T13:27:56,327 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:56,327 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T13:27:56,327 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T13:27:56,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T13:27:56,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T13:27:56,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:56,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:56,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T13:27:56,332 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T13:27:56,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:56,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:56,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T13:27:56,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T13:27:56,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:56,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:56,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T13:27:56,335 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T13:27:56,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:56,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:27:56,336 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T13:27:56,337 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740 2024-12-05T13:27:56,339 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740 2024-12-05T13:27:56,339 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-05T13:27:56,343 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-05T13:27:56,344 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-05T13:27:56,364 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/info/f2d3b9e0770040f4b75858e0dd06d873 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73./info:regioninfo/1733405275328/Put/seqid=0 2024-12-05T13:27:56,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741894_1072 (size=11177) 2024-12-05T13:27:56,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741894_1072 (size=11177) 2024-12-05T13:27:56,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741894_1072 (size=11177) 2024-12-05T13:27:56,372 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/info/f2d3b9e0770040f4b75858e0dd06d873 2024-12-05T13:27:56,395 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/ns/cae23038a03c40208a0d8962ba4d76df is 43, key is default/ns:d/1733405259485/Put/seqid=0 2024-12-05T13:27:56,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741895_1073 (size=5153) 2024-12-05T13:27:56,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741895_1073 (size=5153) 2024-12-05T13:27:56,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741895_1073 (size=5153) 2024-12-05T13:27:56,403 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/ns/cae23038a03c40208a0d8962ba4d76df 2024-12-05T13:27:56,425 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/table/b67d39441a4d44a089f8de1ab3accdd7 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733405273498/Put/seqid=0 2024-12-05T13:27:56,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741896_1074 (size=5431) 2024-12-05T13:27:56,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741896_1074 (size=5431) 2024-12-05T13:27:56,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741896_1074 (size=5431) 2024-12-05T13:27:56,435 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/table/b67d39441a4d44a089f8de1ab3accdd7 2024-12-05T13:27:56,442 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/info/f2d3b9e0770040f4b75858e0dd06d873 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/info/f2d3b9e0770040f4b75858e0dd06d873 2024-12-05T13:27:56,450 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/info/f2d3b9e0770040f4b75858e0dd06d873, entries=36, sequenceid=18, filesize=10.9 K 2024-12-05T13:27:56,451 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/ns/cae23038a03c40208a0d8962ba4d76df as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/ns/cae23038a03c40208a0d8962ba4d76df 2024-12-05T13:27:56,459 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/ns/cae23038a03c40208a0d8962ba4d76df, entries=2, sequenceid=18, filesize=5.0 K 2024-12-05T13:27:56,460 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/table/b67d39441a4d44a089f8de1ab3accdd7 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/table/b67d39441a4d44a089f8de1ab3accdd7 2024-12-05T13:27:56,467 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/table/b67d39441a4d44a089f8de1ab3accdd7, entries=2, sequenceid=18, filesize=5.3 K 2024-12-05T13:27:56,467 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=18, compaction requested=false; wal=null 2024-12-05T13:27:56,468 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-05T13:27:56,469 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T13:27:56,469 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T13:27:56,470 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:27:56,473 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T13:27:56,476 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-05T13:27:56,477 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74432699, jitterRate=0.10913364589214325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:27:56,477 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T13:27:56,478 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733405276327Writing region info on filesystem at 1733405276327Initializing all the Stores at 1733405276328 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405276328Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405276328Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405276328Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733405276328Obtaining lock to block concurrent updates at 1733405276344 (+16 ms)Preparing flush snapshotting stores in 1588230740 at 1733405276344Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733405276344Flushing stores of hbase:meta,,1.1588230740 at 1733405276344Flushing 1588230740/info: creating writer at 1733405276344Flushing 1588230740/info: appending metadata at 1733405276363 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733405276363Flushing 1588230740/ns: creating writer at 1733405276378 (+15 ms)Flushing 1588230740/ns: appending metadata at 1733405276394 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733405276394Flushing 1588230740/table: creating writer at 1733405276409 (+15 ms)Flushing 1588230740/table: appending metadata at 1733405276424 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733405276424Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@148a0950: reopening flushed file at 1733405276441 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fdc0f71: reopening flushed file at 1733405276450 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f55d908: reopening flushed file at 1733405276459 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=18, compaction requested=false; wal=null at 1733405276467 (+8 ms)Cleaning up temporary data from old regions at 1733405276469 (+2 ms)Running coprocessor post-open hooks at 1733405276477 (+8 ms)Region opened successfully at 1733405276478 (+1 ms) 2024-12-05T13:27:56,480 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733405276295 2024-12-05T13:27:56,482 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T13:27:56,483 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T13:27:56,484 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:56,485 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,44873,1733405257343, state=OPEN 2024-12-05T13:27:56,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:56,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:56,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:27:56,512 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:56,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:56,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:56,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:27:56,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-05T13:27:56,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,44873,1733405257343 in 370 msec 2024-12-05T13:27:56,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-05T13:27:56,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 539 msec 2024-12-05T13:27:56,520 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 88aaa70743e7,35441,1733405257297 had 2 regions 2024-12-05T13:27:56,521 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 88aaa70743e7,35441,1733405257297, splitWal=true, meta=true, isMeta: false 2024-12-05T13:27:56,523 INFO [PEWorker-2 {}] master.SplitWALManager(105): 88aaa70743e7,35441,1733405257297 WAL count=1, meta=false 2024-12-05T13:27:56,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 88aaa70743e7%2C35441%2C1733405257297.1733405258866}] 2024-12-05T13:27:56,527 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:56,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C35441%2C1733405257297.1733405258866, worker=88aaa70743e7,44873,1733405257343}] 2024-12-05T13:27:56,550 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:27:56,551 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,44873,1733405257343, seqNum=-1] 2024-12-05T13:27:56,551 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:27:56,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:27:56,669 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-05T13:27:56,669 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-05T13:27:56,670 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 88aaa70743e7:35441 this server is in the failed servers list 2024-12-05T13:27:56,671 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=88aaa70743e7:35441 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 88aaa70743e7:35441 2024-12-05T13:27:56,671 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 88aaa70743e7:35441 2024-12-05T13:27:56,671 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=12 from cache 2024-12-05T13:27:56,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-05T13:27:56,671 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-05T13:27:56,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-05T13:27:56,703 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866, size=2.1 K (2142bytes) 2024-12-05T13:27:56,703 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866 2024-12-05T13:27:56,704 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866 after 1ms 2024-12-05T13:27:56,707 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866 took 4ms 2024-12-05T13:27:56,710 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for d2889c120f533ce016b2cbcb6b393a73: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-05T13:27:56,710 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866 so closing down 2024-12-05T13:27:56,710 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:56,710 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:56,710 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-12-05T13:27:56,710 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866, journal: Splitting hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866, size=2.1 K (2142bytes) at 1733405276703Finishing writing output for hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866 so closing down at 1733405276710 (+7 ms)3 split writer threads finished at 1733405276710Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1733405276710 2024-12-05T13:27:56,711 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866 2024-12-05T13:27:56,711 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-05T13:27:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43665 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-05T13:27:56,716 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting/88aaa70743e7%2C35441%2C1733405257297.1733405258866 to hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs 2024-12-05T13:27:56,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-05T13:27:56,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C35441%2C1733405257297.1733405258866, worker=88aaa70743e7,44873,1733405257343 in 187 msec 2024-12-05T13:27:56,720 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:56,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-05T13:27:56,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C35441%2C1733405257297.1733405258866, worker=88aaa70743e7,44873,1733405257343 in 197 msec 2024-12-05T13:27:56,725 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/WALs/88aaa70743e7,35441,1733405257297-splitting dir is empty, no logs to split. 2024-12-05T13:27:56,725 INFO [PEWorker-2 {}] master.SplitWALManager(105): 88aaa70743e7,35441,1733405257297 WAL count=0, meta=false 2024-12-05T13:27:56,725 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 88aaa70743e7,35441,1733405257297 WAL splitting is done? wals=0, meta=false 2024-12-05T13:27:56,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN}] 2024-12-05T13:27:56,729 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN 2024-12-05T13:27:56,730 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:27:56,880 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:27:56,881 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:27:56,881 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:27:56,881 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:27:56,881 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:27:56,881 INFO [88aaa70743e7:43665 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:27:56,881 INFO [88aaa70743e7:43665 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:27:56,881 DEBUG [88aaa70743e7:43665 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:27:56,882 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPENING, regionLocation=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:56,883 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 88aaa70743e7:35441 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:35441 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:56,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:35441 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:35441 2024-12-05T13:27:56,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-05T13:27:56,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,35441,1733405257297, seqNum=-1 from cache 2024-12-05T13:27:56,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 88aaa70743e7:35441 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:35441 2024-12-05T13:27:56,886 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=18] 2024-12-05T13:27:56,887 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to 88aaa70743e7:35441 this server is in the failed servers list 2024-12-05T13:27:56,887 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=88aaa70743e7:35441 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 88aaa70743e7:35441 2024-12-05T13:27:56,887 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 88aaa70743e7:35441 2024-12-05T13:27:56,887 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,35441,1733405257297, seqNum=18 from cache 2024-12-05T13:27:56,990 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:27:56,990 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,44873,1733405257343, seqNum=-1] 2024-12-05T13:27:56,990 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:27:56,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:27:56,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN because future has completed 2024-12-05T13:27:56,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,44873,1733405257343}] 2024-12-05T13:27:57,153 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:57,153 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => d2889c120f533ce016b2cbcb6b393a73, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:57,154 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,154 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:57,154 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,154 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,156 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,156 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf1 2024-12-05T13:27:57,156 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,163 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf1/2d17b15d7d59457cb38a48ed713a8a89 2024-12-05T13:27:57,164 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,164 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,165 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2889c120f533ce016b2cbcb6b393a73 columnFamilyName cf2 2024-12-05T13:27:57,165 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,172 DEBUG [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/cf2/bc142118035948c1bd0502d9e7d9d1b3 2024-12-05T13:27:57,172 INFO [StoreOpener-d2889c120f533ce016b2cbcb6b393a73-1 {}] regionserver.HStore(327): Store=d2889c120f533ce016b2cbcb6b393a73/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,172 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,173 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,174 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,175 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,175 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,176 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-05T13:27:57,177 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,178 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened d2889c120f533ce016b2cbcb6b393a73; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68487558, jitterRate=0.0205441415309906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-05T13:27:57,178 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:27:57,178 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for d2889c120f533ce016b2cbcb6b393a73: Running coprocessor pre-open hook at 1733405277154Writing region info on filesystem at 1733405277154Initializing all the Stores at 1733405277155 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277155Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277155Cleaning up temporary data from old regions at 1733405277175 (+20 ms)Running coprocessor post-open hooks at 1733405277178 (+3 ms)Region opened successfully at 1733405277178 2024-12-05T13:27:57,180 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., pid=21, masterSystemTime=1733405277148 2024-12-05T13:27:57,182 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:57,182 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:27:57,183 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d2889c120f533ce016b2cbcb6b393a73, regionState=OPEN, openSeqNum=18, regionLocation=88aaa70743e7,44873,1733405257343 2024-12-05T13:27:57,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,44873,1733405257343 because future has completed 2024-12-05T13:27:57,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-05T13:27:57,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d2889c120f533ce016b2cbcb6b393a73, server=88aaa70743e7,44873,1733405257343 in 191 msec 2024-12-05T13:27:57,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-05T13:27:57,191 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 88aaa70743e7,35441,1733405257297 after splitting done 2024-12-05T13:27:57,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=d2889c120f533ce016b2cbcb6b393a73, ASSIGN in 461 msec 2024-12-05T13:27:57,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 88aaa70743e7,35441,1733405257297, splitWal=true, meta=true in 1.4780 sec 2024-12-05T13:27:57,194 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73., hostname=88aaa70743e7,44873,1733405257343, seqNum=18] 2024-12-05T13:27:57,209 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=407 (was 412), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=243 (was 212) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 12), AvailableMemoryMB=4138 (was 4227) 2024-12-05T13:27:57,211 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-05T13:27:57,225 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=407, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=243, ProcessCount=12, AvailableMemoryMB=4137 2024-12-05T13:27:57,225 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-05T13:27:57,246 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:57,249 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:57,250 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:57,253 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58132387, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-58132387, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:57,268 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58132387/hregion-58132387.1733405277254, exclude list is [], retry=0 2024-12-05T13:27:57,271 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:57,272 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:57,275 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:57,285 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58132387/hregion-58132387.1733405277254 2024-12-05T13:27:57,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:57,286 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cfcd1100e4282a5fd364d46c7fbbfb66, NAME => 'testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:27:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741898_1076 (size=67) 2024-12-05T13:27:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741898_1076 (size=67) 2024-12-05T13:27:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741898_1076 (size=67) 2024-12-05T13:27:57,306 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:57,307 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,309 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName a 2024-12-05T13:27:57,309 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,310 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,310 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,312 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName b 2024-12-05T13:27:57,312 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,312 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,312 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,314 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName c 2024-12-05T13:27:57,314 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,314 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,315 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,316 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,316 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,317 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,317 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:57,319 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,321 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:57,322 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cfcd1100e4282a5fd364d46c7fbbfb66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72992530, jitterRate=0.08767345547676086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:57,323 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cfcd1100e4282a5fd364d46c7fbbfb66: Writing region info on filesystem at 1733405277306Initializing all the Stores at 1733405277307 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277307Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277307Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277307Cleaning up temporary data from old regions at 1733405277317 (+10 ms)Region opened successfully at 1733405277323 (+6 ms) 2024-12-05T13:27:57,323 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cfcd1100e4282a5fd364d46c7fbbfb66, disabling compactions & flushes 2024-12-05T13:27:57,323 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,323 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,323 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. after waiting 0 ms 2024-12-05T13:27:57,323 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,330 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,330 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cfcd1100e4282a5fd364d46c7fbbfb66: Waiting for close lock at 1733405277323Disabling compacts and flushes for region at 1733405277323Disabling writes for close at 1733405277323Writing region close event to WAL at 1733405277330 (+7 ms)Closed at 1733405277330 2024-12-05T13:27:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741897_1075 (size=93) 2024-12-05T13:27:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741897_1075 (size=93) 2024-12-05T13:27:57,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741897_1075 (size=93) 2024-12-05T13:27:57,338 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-58132387/hregion-58132387.1733405277254 not finished, retry = 0 2024-12-05T13:27:57,441 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:57,441 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58132387:(num 1733405277254) 2024-12-05T13:27:57,441 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:57,444 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:57,457 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444, exclude list is [], retry=0 2024-12-05T13:27:57,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:57,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:57,460 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:57,462 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444 2024-12-05T13:27:57,462 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:27:57,462 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cfcd1100e4282a5fd364d46c7fbbfb66, NAME => 'testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:57,462 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:57,463 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,463 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,465 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,466 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName a 2024-12-05T13:27:57,466 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,466 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,467 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,467 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName b 2024-12-05T13:27:57,467 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,468 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,468 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,469 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName c 2024-12-05T13:27:57,469 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,470 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,470 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,472 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,473 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,473 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,473 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:57,476 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,476 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cfcd1100e4282a5fd364d46c7fbbfb66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63240408, jitterRate=-0.05764448642730713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:57,477 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cfcd1100e4282a5fd364d46c7fbbfb66: Writing region info on filesystem at 1733405277463Initializing all the Stores at 1733405277464 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277464Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277464Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277464Cleaning up temporary data from old regions at 1733405277473 (+9 ms)Region opened successfully at 1733405277477 (+4 ms) 2024-12-05T13:27:57,515 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cfcd1100e4282a5fd364d46c7fbbfb66 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-05T13:27:57,535 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/a/03a35e1ac6c1474abeef7919f4aec05e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733405277477/Put/seqid=0 2024-12-05T13:27:57,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741900_1078 (size=5958) 2024-12-05T13:27:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741900_1078 (size=5958) 2024-12-05T13:27:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741900_1078 (size=5958) 2024-12-05T13:27:57,544 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/a/03a35e1ac6c1474abeef7919f4aec05e 2024-12-05T13:27:57,567 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/b/6ac607c4d61b4740a0ec46a7d849e373 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733405277489/Put/seqid=0 2024-12-05T13:27:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741901_1079 (size=5958) 2024-12-05T13:27:57,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741901_1079 (size=5958) 2024-12-05T13:27:57,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741901_1079 (size=5958) 2024-12-05T13:27:57,576 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/b/6ac607c4d61b4740a0ec46a7d849e373 2024-12-05T13:27:57,597 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/c/46893a6515e241ddb9a2d6f69fa3f9d0 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733405277503/Put/seqid=0 2024-12-05T13:27:57,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741902_1080 (size=5958) 2024-12-05T13:27:57,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741902_1080 (size=5958) 2024-12-05T13:27:57,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741902_1080 (size=5958) 2024-12-05T13:27:57,606 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/c/46893a6515e241ddb9a2d6f69fa3f9d0 2024-12-05T13:27:57,613 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/a/03a35e1ac6c1474abeef7919f4aec05e as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/a/03a35e1ac6c1474abeef7919f4aec05e 2024-12-05T13:27:57,620 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/a/03a35e1ac6c1474abeef7919f4aec05e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-05T13:27:57,622 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/b/6ac607c4d61b4740a0ec46a7d849e373 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/b/6ac607c4d61b4740a0ec46a7d849e373 2024-12-05T13:27:57,628 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/b/6ac607c4d61b4740a0ec46a7d849e373, entries=10, sequenceid=33, filesize=5.8 K 2024-12-05T13:27:57,630 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/c/46893a6515e241ddb9a2d6f69fa3f9d0 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/c/46893a6515e241ddb9a2d6f69fa3f9d0 2024-12-05T13:27:57,636 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/c/46893a6515e241ddb9a2d6f69fa3f9d0, entries=10, sequenceid=33, filesize=5.8 K 2024-12-05T13:27:57,638 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for cfcd1100e4282a5fd364d46c7fbbfb66 in 123ms, sequenceid=33, compaction requested=false 2024-12-05T13:27:57,638 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cfcd1100e4282a5fd364d46c7fbbfb66: 2024-12-05T13:27:57,638 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cfcd1100e4282a5fd364d46c7fbbfb66, disabling compactions & flushes 2024-12-05T13:27:57,638 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,638 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,638 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. after waiting 0 ms 2024-12-05T13:27:57,638 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,639 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. 2024-12-05T13:27:57,639 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cfcd1100e4282a5fd364d46c7fbbfb66: Waiting for close lock at 1733405277638Disabling compacts and flushes for region at 1733405277638Disabling writes for close at 1733405277638Writing region close event to WAL at 1733405277639 (+1 ms)Closed at 1733405277639 2024-12-05T13:27:57,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741899_1077 (size=5937) 2024-12-05T13:27:57,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741899_1077 (size=5937) 2024-12-05T13:27:57,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741899_1077 (size=5937) 2024-12-05T13:27:57,649 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/b/6ac607c4d61b4740a0ec46a7d849e373 to hdfs://localhost:44203/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/b/6ac607c4d61b4740a0ec46a7d849e373 2024-12-05T13:27:57,667 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444, size=5.8 K (5937bytes) 2024-12-05T13:27:57,667 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444 2024-12-05T13:27:57,667 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444 after 0ms 2024-12-05T13:27:57,670 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444 took 3ms 2024-12-05T13:27:57,672 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444 so closing down 2024-12-05T13:27:57,673 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:27:57,675 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733405277444.temp 2024-12-05T13:27:57,676 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000003-wal.1733405277444.temp 2024-12-05T13:27:57,677 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:27:57,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741903_1081 (size=5223) 2024-12-05T13:27:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741903_1081 (size=5223) 2024-12-05T13:27:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741903_1081 (size=5223) 2024-12-05T13:27:57,691 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000003-wal.1733405277444.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-05T13:27:57,693 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000003-wal.1733405277444.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000032 2024-12-05T13:27:57,693 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 23 ms; skipped=2; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-12-05T13:27:57,693 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444, journal: Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444, size=5.8 K (5937bytes) at 1733405277667Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444 so closing down at 1733405277672 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000003-wal.1733405277444.temp at 1733405277676 (+4 ms)3 split writer threads finished at 1733405277677 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000003-wal.1733405277444.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733405277691 (+14 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000003-wal.1733405277444.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000032 at 1733405277693 (+2 ms)Processed 32 edits across 1 Regions in 23 ms; skipped=2; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1733405277693 2024-12-05T13:27:57,695 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277444 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405277444 2024-12-05T13:27:57,696 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000032 2024-12-05T13:27:57,696 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:57,697 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:57,713 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277698, exclude list is [], retry=0 2024-12-05T13:27:57,717 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:57,717 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:57,718 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:57,719 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277698 2024-12-05T13:27:57,720 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:57,720 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cfcd1100e4282a5fd364d46c7fbbfb66, NAME => 'testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:57,720 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:57,720 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,720 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,721 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,722 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName a 2024-12-05T13:27:57,722 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,730 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/a/03a35e1ac6c1474abeef7919f4aec05e 2024-12-05T13:27:57,730 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,730 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,732 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName b 2024-12-05T13:27:57,732 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,732 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,733 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,734 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfcd1100e4282a5fd364d46c7fbbfb66 columnFamilyName c 2024-12-05T13:27:57,734 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,740 DEBUG [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/c/46893a6515e241ddb9a2d6f69fa3f9d0 2024-12-05T13:27:57,740 INFO [StoreOpener-cfcd1100e4282a5fd364d46c7fbbfb66-1 {}] regionserver.HStore(327): Store=cfcd1100e4282a5fd364d46c7fbbfb66/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,740 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,743 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,743 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000032 2024-12-05T13:27:57,747 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000032 2024-12-05T13:27:57,747 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cfcd1100e4282a5fd364d46c7fbbfb66 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-05T13:27:57,764 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/b/9e15fb7a96044d4a9d1607903a6fa230 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733405277489/Put/seqid=0 2024-12-05T13:27:57,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741905_1083 (size=5958) 2024-12-05T13:27:57,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741905_1083 (size=5958) 2024-12-05T13:27:57,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741905_1083 (size=5958) 2024-12-05T13:27:57,774 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/b/9e15fb7a96044d4a9d1607903a6fa230 2024-12-05T13:27:57,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/.tmp/b/9e15fb7a96044d4a9d1607903a6fa230 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/b/9e15fb7a96044d4a9d1607903a6fa230 2024-12-05T13:27:57,786 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/b/9e15fb7a96044d4a9d1607903a6fa230, entries=10, sequenceid=32, filesize=5.8 K 2024-12-05T13:27:57,786 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for cfcd1100e4282a5fd364d46c7fbbfb66 in 39ms, sequenceid=32, compaction requested=false; wal=null 2024-12-05T13:27:57,787 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/0000000000000000032 2024-12-05T13:27:57,789 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,789 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,790 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:57,792 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cfcd1100e4282a5fd364d46c7fbbfb66 2024-12-05T13:27:57,794 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/cfcd1100e4282a5fd364d46c7fbbfb66/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-05T13:27:57,796 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cfcd1100e4282a5fd364d46c7fbbfb66; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70662323, jitterRate=0.0529506653547287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:57,796 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cfcd1100e4282a5fd364d46c7fbbfb66: Writing region info on filesystem at 1733405277720Initializing all the Stores at 1733405277721 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277721Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277721Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277721Obtaining lock to block concurrent updates at 1733405277747 (+26 ms)Preparing flush snapshotting stores in cfcd1100e4282a5fd364d46c7fbbfb66 at 1733405277747Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733405277747Flushing stores of testReplayEditsWrittenViaHRegion,,1733405277247.cfcd1100e4282a5fd364d46c7fbbfb66. at 1733405277747Flushing cfcd1100e4282a5fd364d46c7fbbfb66/b: creating writer at 1733405277747Flushing cfcd1100e4282a5fd364d46c7fbbfb66/b: appending metadata at 1733405277763 (+16 ms)Flushing cfcd1100e4282a5fd364d46c7fbbfb66/b: closing flushed file at 1733405277763Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@264805fc: reopening flushed file at 1733405277779 (+16 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for cfcd1100e4282a5fd364d46c7fbbfb66 in 39ms, sequenceid=32, compaction requested=false; wal=null at 1733405277787 (+8 ms)Cleaning up temporary data from old regions at 1733405277789 (+2 ms)Region opened successfully at 1733405277796 (+7 ms) 2024-12-05T13:27:57,818 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=416 (was 407) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:53574 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33184 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:36266 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:49454 [Waiting for operation #49] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:42190 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:36690 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=243 (was 243), ProcessCount=11 (was 12), AvailableMemoryMB=4122 (was 4137) 2024-12-05T13:27:57,818 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-05T13:27:57,830 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=416, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=243, ProcessCount=11, AvailableMemoryMB=4122 2024-12-05T13:27:57,830 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-05T13:27:57,843 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:57,846 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:27:57,846 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:27:57,849 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22219842, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-22219842, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:57,863 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22219842/hregion-22219842.1733405277849, exclude list is [], retry=0 2024-12-05T13:27:57,866 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:57,866 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:57,866 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:57,868 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22219842/hregion-22219842.1733405277849 2024-12-05T13:27:57,868 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:27:57,869 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 96f1d02a4f4f5510d50e99ec074a5309, NAME => 'testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:27:57,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741907_1085 (size=68) 2024-12-05T13:27:57,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741907_1085 (size=68) 2024-12-05T13:27:57,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741907_1085 (size=68) 2024-12-05T13:27:57,877 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:57,879 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,880 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName a 2024-12-05T13:27:57,881 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,881 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,881 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,883 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName b 2024-12-05T13:27:57,883 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,883 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,883 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,885 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName c 2024-12-05T13:27:57,885 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,886 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,886 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,887 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,887 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,888 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,888 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,889 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:57,890 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,892 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:27:57,892 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 96f1d02a4f4f5510d50e99ec074a5309; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63114038, jitterRate=-0.05952754616737366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:57,893 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 96f1d02a4f4f5510d50e99ec074a5309: Writing region info on filesystem at 1733405277877Initializing all the Stores at 1733405277878 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277878Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277878Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277878Cleaning up temporary data from old regions at 1733405277888 (+10 ms)Region opened successfully at 1733405277892 (+4 ms) 2024-12-05T13:27:57,893 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 96f1d02a4f4f5510d50e99ec074a5309, disabling compactions & flushes 2024-12-05T13:27:57,893 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:27:57,893 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:27:57,893 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. after waiting 0 ms 2024-12-05T13:27:57,893 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:27:57,893 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:27:57,893 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 96f1d02a4f4f5510d50e99ec074a5309: Waiting for close lock at 1733405277893Disabling compacts and flushes for region at 1733405277893Disabling writes for close at 1733405277893Writing region close event to WAL at 1733405277893Closed at 1733405277893 2024-12-05T13:27:57,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741906_1084 (size=93) 2024-12-05T13:27:57,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741906_1084 (size=93) 2024-12-05T13:27:57,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741906_1084 (size=93) 2024-12-05T13:27:57,899 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:27:57,899 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22219842:(num 1733405277849) 2024-12-05T13:27:57,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:27:57,900 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:27:57,914 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901, exclude list is [], retry=0 2024-12-05T13:27:57,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:27:57,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:27:57,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:27:57,918 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901 2024-12-05T13:27:57,919 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:27:57,978 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 96f1d02a4f4f5510d50e99ec074a5309, NAME => 'testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:27:57,981 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,981 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:27:57,981 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,981 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,983 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,985 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName a 2024-12-05T13:27:57,985 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,986 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,986 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,987 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName b 2024-12-05T13:27:57,987 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,988 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,989 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,990 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName c 2024-12-05T13:27:57,990 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:27:57,991 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:27:57,991 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,992 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,993 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,994 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,994 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,995 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:27:57,996 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,997 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 96f1d02a4f4f5510d50e99ec074a5309; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66291114, jitterRate=-0.012185424566268921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:27:57,997 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:27:57,997 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 96f1d02a4f4f5510d50e99ec074a5309: Running coprocessor pre-open hook at 1733405277982Writing region info on filesystem at 1733405277982Initializing all the Stores at 1733405277983 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277983Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277983Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405277983Cleaning up temporary data from old regions at 1733405277994 (+11 ms)Running coprocessor post-open hooks at 1733405277997 (+3 ms)Region opened successfully at 1733405277997 2024-12-05T13:27:58,011 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 96f1d02a4f4f5510d50e99ec074a5309 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-05T13:27:58,012 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:27:58,990 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T13:27:59,013 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:00,014 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:01,014 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:02,015 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:03,016 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:04,017 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:05,018 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:05,512 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T13:28:06,019 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:06,669 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-05T13:28:06,669 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-05T13:28:06,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-05T13:28:06,671 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-05T13:28:06,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-05T13:28:06,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-05T13:28:07,019 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 96f1d02a4f4f5510d50e99ec074a5309/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:07,021 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 96f1d02a4f4f5510d50e99ec074a5309: 2024-12-05T13:28:07,021 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:28:07,036 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 96f1d02a4f4f5510d50e99ec074a5309: 2024-12-05T13:28:07,036 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-05T13:28:07,036 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 96f1d02a4f4f5510d50e99ec074a5309, disabling compactions & flushes 2024-12-05T13:28:07,036 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:28:07,036 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:28:07,036 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. after waiting 0 ms 2024-12-05T13:28:07,036 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:28:07,037 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:28:07,037 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. 2024-12-05T13:28:07,037 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 96f1d02a4f4f5510d50e99ec074a5309: Waiting for close lock at 1733405287036Running coprocessor pre-close hooks at 1733405287036Disabling compacts and flushes for region at 1733405287036Disabling writes for close at 1733405287036Writing region close event to WAL at 1733405287037 (+1 ms)Running coprocessor post-close hooks at 1733405287037Closed at 1733405287037 2024-12-05T13:28:07,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741908_1086 (size=3941) 2024-12-05T13:28:07,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741908_1086 (size=3941) 2024-12-05T13:28:07,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741908_1086 (size=3941) 2024-12-05T13:28:07,056 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901, size=3.8 K (3941bytes) 2024-12-05T13:28:07,056 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901 2024-12-05T13:28:07,056 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901 after 0ms 2024-12-05T13:28:07,059 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901 took 3ms 2024-12-05T13:28:07,061 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901 so closing down 2024-12-05T13:28:07,061 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:28:07,062 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733405277901.temp 2024-12-05T13:28:07,064 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000004-wal.1733405277901.temp 2024-12-05T13:28:07,064 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:28:07,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741909_1087 (size=2983) 2024-12-05T13:28:07,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741909_1087 (size=2983) 2024-12-05T13:28:07,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741909_1087 (size=2983) 2024-12-05T13:28:07,072 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000004-wal.1733405277901.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-05T13:28:07,074 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000004-wal.1733405277901.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000026 2024-12-05T13:28:07,074 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-12-05T13:28:07,074 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901, journal: Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901, size=3.8 K (3941bytes) at 1733405287056Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901 so closing down at 1733405287061 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000004-wal.1733405277901.temp at 1733405287064 (+3 ms)3 split writer threads finished at 1733405287064Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000004-wal.1733405277901.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733405287072 (+8 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000004-wal.1733405277901.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000026 at 1733405287074 (+2 ms)Processed 23 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1733405287074 2024-12-05T13:28:07,076 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405277901 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405277901 2024-12-05T13:28:07,077 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000026 2024-12-05T13:28:07,077 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:07,079 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:07,092 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405287079, exclude list is [], retry=0 2024-12-05T13:28:07,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:07,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:07,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:07,099 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405287079 2024-12-05T13:28:07,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:28:07,100 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 96f1d02a4f4f5510d50e99ec074a5309, NAME => 'testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:07,101 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,101 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:07,101 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,101 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,103 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,104 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName a 2024-12-05T13:28:07,104 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:07,105 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:07,105 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,106 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName b 2024-12-05T13:28:07,106 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:07,107 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:07,107 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,108 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96f1d02a4f4f5510d50e99ec074a5309 columnFamilyName c 2024-12-05T13:28:07,108 DEBUG [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:07,109 INFO [StoreOpener-96f1d02a4f4f5510d50e99ec074a5309-1 {}] regionserver.HStore(327): Store=96f1d02a4f4f5510d50e99ec074a5309/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:07,109 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,109 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,111 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,111 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000026 2024-12-05T13:28:07,115 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000026 2024-12-05T13:28:07,115 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 96f1d02a4f4f5510d50e99ec074a5309 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-05T13:28:07,131 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/a/d47bda0487f14348b6a9fec7dc486040 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733405287027/Put/seqid=0 2024-12-05T13:28:07,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741911_1089 (size=5523) 2024-12-05T13:28:07,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741911_1089 (size=5523) 2024-12-05T13:28:07,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741911_1089 (size=5523) 2024-12-05T13:28:07,143 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/a/d47bda0487f14348b6a9fec7dc486040 2024-12-05T13:28:07,165 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/b/6487ef622c2d4749a3686f5bc4d10eb1 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733405287022/Put/seqid=0 2024-12-05T13:28:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741912_1090 (size=5524) 2024-12-05T13:28:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741912_1090 (size=5524) 2024-12-05T13:28:07,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741912_1090 (size=5524) 2024-12-05T13:28:07,173 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/b/6487ef622c2d4749a3686f5bc4d10eb1 2024-12-05T13:28:07,194 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/c/88b723bb590840e69a4a80901302e819 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733405287024/Put/seqid=0 2024-12-05T13:28:07,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741913_1091 (size=5457) 2024-12-05T13:28:07,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741913_1091 (size=5457) 2024-12-05T13:28:07,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741913_1091 (size=5457) 2024-12-05T13:28:07,206 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/c/88b723bb590840e69a4a80901302e819 2024-12-05T13:28:07,214 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/a/d47bda0487f14348b6a9fec7dc486040 as hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/a/d47bda0487f14348b6a9fec7dc486040 2024-12-05T13:28:07,222 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/a/d47bda0487f14348b6a9fec7dc486040, entries=7, sequenceid=26, filesize=5.4 K 2024-12-05T13:28:07,223 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/b/6487ef622c2d4749a3686f5bc4d10eb1 as hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/b/6487ef622c2d4749a3686f5bc4d10eb1 2024-12-05T13:28:07,229 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/b/6487ef622c2d4749a3686f5bc4d10eb1, entries=7, sequenceid=26, filesize=5.4 K 2024-12-05T13:28:07,231 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/.tmp/c/88b723bb590840e69a4a80901302e819 as hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/c/88b723bb590840e69a4a80901302e819 2024-12-05T13:28:07,238 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/c/88b723bb590840e69a4a80901302e819, entries=6, sequenceid=26, filesize=5.3 K 2024-12-05T13:28:07,238 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 96f1d02a4f4f5510d50e99ec074a5309 in 123ms, sequenceid=26, compaction requested=false; wal=null 2024-12-05T13:28:07,239 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/0000000000000000026 2024-12-05T13:28:07,241 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,241 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,242 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:07,243 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,246 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsAfterAbortingFlush/96f1d02a4f4f5510d50e99ec074a5309/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-05T13:28:07,247 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 96f1d02a4f4f5510d50e99ec074a5309; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59599254, jitterRate=-0.1119019091129303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:07,247 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96f1d02a4f4f5510d50e99ec074a5309 2024-12-05T13:28:07,247 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 96f1d02a4f4f5510d50e99ec074a5309: Running coprocessor pre-open hook at 1733405287101Writing region info on filesystem at 1733405287101Initializing all the Stores at 1733405287102 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405287102Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405287102Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405287102Obtaining lock to block concurrent updates at 1733405287115 (+13 ms)Preparing flush snapshotting stores in 96f1d02a4f4f5510d50e99ec074a5309 at 1733405287115Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733405287115Flushing stores of testReplayEditsAfterAbortingFlush,,1733405277844.96f1d02a4f4f5510d50e99ec074a5309. at 1733405287115Flushing 96f1d02a4f4f5510d50e99ec074a5309/a: creating writer at 1733405287116 (+1 ms)Flushing 96f1d02a4f4f5510d50e99ec074a5309/a: appending metadata at 1733405287130 (+14 ms)Flushing 96f1d02a4f4f5510d50e99ec074a5309/a: closing flushed file at 1733405287130Flushing 96f1d02a4f4f5510d50e99ec074a5309/b: creating writer at 1733405287150 (+20 ms)Flushing 96f1d02a4f4f5510d50e99ec074a5309/b: appending metadata at 1733405287165 (+15 ms)Flushing 96f1d02a4f4f5510d50e99ec074a5309/b: closing flushed file at 1733405287165Flushing 96f1d02a4f4f5510d50e99ec074a5309/c: creating writer at 1733405287179 (+14 ms)Flushing 96f1d02a4f4f5510d50e99ec074a5309/c: appending metadata at 1733405287193 (+14 ms)Flushing 96f1d02a4f4f5510d50e99ec074a5309/c: closing flushed file at 1733405287193Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14b1b309: reopening flushed file at 1733405287213 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44af1a71: reopening flushed file at 1733405287222 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e6d516b: reopening flushed file at 1733405287229 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 96f1d02a4f4f5510d50e99ec074a5309 in 123ms, sequenceid=26, compaction requested=false; wal=null at 1733405287238 (+9 ms)Cleaning up temporary data from old regions at 1733405287241 (+3 ms)Running coprocessor post-open hooks at 1733405287247 (+6 ms)Region opened successfully at 1733405287247 2024-12-05T13:28:07,269 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417 (was 416) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33136 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:43492 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:43466 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:35836 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33160 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1180 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 243), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=4005 (was 4122) 2024-12-05T13:28:07,269 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-05T13:28:07,285 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=12, AvailableMemoryMB=4004 2024-12-05T13:28:07,285 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-05T13:28:07,301 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:28:07,303 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:28:07,303 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:28:07,306 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-61787877, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-61787877, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:07,323 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-61787877/hregion-61787877.1733405287306, exclude list is [], retry=0 2024-12-05T13:28:07,327 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:07,328 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:07,329 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:07,336 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-61787877/hregion-61787877.1733405287306 2024-12-05T13:28:07,339 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:28:07,339 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 461d15a5902b518b0f3344467640ed04, NAME => 'testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:28:07,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741915_1093 (size=61) 2024-12-05T13:28:07,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741915_1093 (size=61) 2024-12-05T13:28:07,355 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:07,357 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741915_1093 (size=61) 2024-12-05T13:28:07,359 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 461d15a5902b518b0f3344467640ed04 columnFamilyName a 2024-12-05T13:28:07,359 DEBUG [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:07,360 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(327): Store=461d15a5902b518b0f3344467640ed04/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:07,360 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,361 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,361 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,362 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,362 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,363 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,366 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:28:07,366 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 461d15a5902b518b0f3344467640ed04; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60978826, jitterRate=-0.0913446843624115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:28:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 461d15a5902b518b0f3344467640ed04: Writing region info on filesystem at 1733405287355Initializing all the Stores at 1733405287356 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405287356Cleaning up temporary data from old regions at 1733405287362 (+6 ms)Region opened successfully at 1733405287367 (+5 ms) 2024-12-05T13:28:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 461d15a5902b518b0f3344467640ed04, disabling compactions & flushes 2024-12-05T13:28:07,367 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. after waiting 0 ms 2024-12-05T13:28:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,367 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,367 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 461d15a5902b518b0f3344467640ed04: Waiting for close lock at 1733405287367Disabling compacts and flushes for region at 1733405287367Disabling writes for close at 1733405287367Writing region close event to WAL at 1733405287367Closed at 1733405287367 2024-12-05T13:28:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741914_1092 (size=93) 2024-12-05T13:28:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741914_1092 (size=93) 2024-12-05T13:28:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741914_1092 (size=93) 2024-12-05T13:28:07,372 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:28:07,372 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-61787877:(num 1733405287306) 2024-12-05T13:28:07,372 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:07,374 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:07,387 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374, exclude list is [], retry=0 2024-12-05T13:28:07,390 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:07,390 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:07,391 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:07,403 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374 2024-12-05T13:28:07,403 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:28:07,403 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 461d15a5902b518b0f3344467640ed04, NAME => 'testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:07,403 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:07,404 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,404 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,405 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,406 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 461d15a5902b518b0f3344467640ed04 columnFamilyName a 2024-12-05T13:28:07,406 DEBUG [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:07,407 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(327): Store=461d15a5902b518b0f3344467640ed04/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:07,407 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,410 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,410 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,411 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,412 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 461d15a5902b518b0f3344467640ed04; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62561983, jitterRate=-0.06775380671024323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:28:07,413 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 461d15a5902b518b0f3344467640ed04: Writing region info on filesystem at 1733405287404Initializing all the Stores at 1733405287404Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405287404Cleaning up temporary data from old regions at 1733405287410 (+6 ms)Region opened successfully at 1733405287412 (+2 ms) 2024-12-05T13:28:07,421 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 461d15a5902b518b0f3344467640ed04, disabling compactions & flushes 2024-12-05T13:28:07,421 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,421 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,421 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. after waiting 0 ms 2024-12-05T13:28:07,421 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,422 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,422 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. 2024-12-05T13:28:07,422 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 461d15a5902b518b0f3344467640ed04: Waiting for close lock at 1733405287421Disabling compacts and flushes for region at 1733405287421Disabling writes for close at 1733405287421Writing region close event to WAL at 1733405287422 (+1 ms)Closed at 1733405287422 2024-12-05T13:28:07,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741916_1094 (size=1623) 2024-12-05T13:28:07,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741916_1094 (size=1623) 2024-12-05T13:28:07,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741916_1094 (size=1623) 2024-12-05T13:28:07,442 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374, size=1.6 K (1623bytes) 2024-12-05T13:28:07,442 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374 2024-12-05T13:28:07,443 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374 after 1ms 2024-12-05T13:28:07,445 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374 took 3ms 2024-12-05T13:28:07,447 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374 so closing down 2024-12-05T13:28:07,447 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:28:07,448 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733405287374.temp 2024-12-05T13:28:07,450 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000003-wal.1733405287374.temp 2024-12-05T13:28:07,450 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:28:07,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741917_1095 (size=1623) 2024-12-05T13:28:07,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741917_1095 (size=1623) 2024-12-05T13:28:07,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741917_1095 (size=1623) 2024-12-05T13:28:07,456 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000003-wal.1733405287374.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-05T13:28:07,457 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000003-wal.1733405287374.temp to hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000012 2024-12-05T13:28:07,458 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-12-05T13:28:07,458 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374, journal: Splitting hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374, size=1.6 K (1623bytes) at 1733405287442Finishing writing output for hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374 so closing down at 1733405287447 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000003-wal.1733405287374.temp at 1733405287450 (+3 ms)3 split writer threads finished at 1733405287450Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000003-wal.1733405287374.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733405287456 (+6 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000003-wal.1733405287374.temp to hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000012 at 1733405287457 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1733405287458 (+1 ms) 2024-12-05T13:28:07,459 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287374 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405287374 2024-12-05T13:28:07,460 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000012 2024-12-05T13:28:07,842 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:07,845 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:07,857 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287845, exclude list is [], retry=0 2024-12-05T13:28:07,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:07,860 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:07,860 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:07,861 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287845 2024-12-05T13:28:07,861 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653)] 2024-12-05T13:28:07,862 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 461d15a5902b518b0f3344467640ed04, NAME => 'testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:07,862 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:07,862 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,862 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,865 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,867 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 461d15a5902b518b0f3344467640ed04 columnFamilyName a 2024-12-05T13:28:07,867 DEBUG [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:07,868 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(327): Store=461d15a5902b518b0f3344467640ed04/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:07,868 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,869 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,870 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,871 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000012 2024-12-05T13:28:07,874 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000012 2024-12-05T13:28:07,874 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 461d15a5902b518b0f3344467640ed04 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-05T13:28:07,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/.tmp/a/a743c4206eab4d5bbff34c1041f6f8b9 is 79, key is testDatalossWhenInputError/a:x0/1733405287413/Put/seqid=0 2024-12-05T13:28:07,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741919_1097 (size=5808) 2024-12-05T13:28:07,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741919_1097 (size=5808) 2024-12-05T13:28:07,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741919_1097 (size=5808) 2024-12-05T13:28:07,903 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/.tmp/a/a743c4206eab4d5bbff34c1041f6f8b9 2024-12-05T13:28:07,914 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/.tmp/a/a743c4206eab4d5bbff34c1041f6f8b9 as hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/a/a743c4206eab4d5bbff34c1041f6f8b9 2024-12-05T13:28:07,924 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/a/a743c4206eab4d5bbff34c1041f6f8b9, entries=10, sequenceid=12, filesize=5.7 K 2024-12-05T13:28:07,924 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 461d15a5902b518b0f3344467640ed04 in 50ms, sequenceid=12, compaction requested=false; wal=null 2024-12-05T13:28:07,925 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/0000000000000000012 2024-12-05T13:28:07,926 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,926 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,929 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,931 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-05T13:28:07,932 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 461d15a5902b518b0f3344467640ed04; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69456204, jitterRate=0.03497809171676636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:28:07,933 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 461d15a5902b518b0f3344467640ed04: Writing region info on filesystem at 1733405287862Initializing all the Stores at 1733405287865 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405287865Obtaining lock to block concurrent updates at 1733405287874 (+9 ms)Preparing flush snapshotting stores in 461d15a5902b518b0f3344467640ed04 at 1733405287874Finished memstore snapshotting testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733405287874Flushing stores of testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04. at 1733405287874Flushing 461d15a5902b518b0f3344467640ed04/a: creating writer at 1733405287874Flushing 461d15a5902b518b0f3344467640ed04/a: appending metadata at 1733405287895 (+21 ms)Flushing 461d15a5902b518b0f3344467640ed04/a: closing flushed file at 1733405287895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@569a0ae: reopening flushed file at 1733405287912 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 461d15a5902b518b0f3344467640ed04 in 50ms, sequenceid=12, compaction requested=false; wal=null at 1733405287924 (+12 ms)Cleaning up temporary data from old regions at 1733405287926 (+2 ms)Region opened successfully at 1733405287933 (+7 ms) 2024-12-05T13:28:07,936 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 461d15a5902b518b0f3344467640ed04, NAME => 'testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:07,936 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733405287301.461d15a5902b518b0f3344467640ed04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:07,936 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,936 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,938 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,939 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 461d15a5902b518b0f3344467640ed04 columnFamilyName a 2024-12-05T13:28:07,939 DEBUG [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:07,944 DEBUG [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/a/a743c4206eab4d5bbff34c1041f6f8b9 2024-12-05T13:28:07,944 INFO [StoreOpener-461d15a5902b518b0f3344467640ed04-1 {}] regionserver.HStore(327): Store=461d15a5902b518b0f3344467640ed04/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:07,945 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,945 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,946 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,947 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,947 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,948 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 461d15a5902b518b0f3344467640ed04 2024-12-05T13:28:07,951 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testDatalossWhenInputError/461d15a5902b518b0f3344467640ed04/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-05T13:28:07,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 461d15a5902b518b0f3344467640ed04; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64430448, jitterRate=-0.039911508560180664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:28:07,952 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 461d15a5902b518b0f3344467640ed04: Writing region info on filesystem at 1733405287937Initializing all the Stores at 1733405287937Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405287937Cleaning up temporary data from old regions at 1733405287947 (+10 ms)Region opened successfully at 1733405287952 (+5 ms) 2024-12-05T13:28:07,969 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=428 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33136 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:35896 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:35914 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:43492 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1492241733-172.17.0.3-1733405252624:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33222 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:43570 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 228), ProcessCount=11 (was 12), AvailableMemoryMB=3984 (was 4004) 2024-12-05T13:28:07,969 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-05T13:28:07,983 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=3982 2024-12-05T13:28:07,983 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-05T13:28:07,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:28:08,001 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:28:08,002 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:28:08,004 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-64332669, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-64332669, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:08,017 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-64332669/hregion-64332669.1733405288005, exclude list is [], retry=0 2024-12-05T13:28:08,020 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:08,021 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:08,021 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:08,025 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-64332669/hregion-64332669.1733405288005 2024-12-05T13:28:08,026 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:28:08,026 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 75e0611148ca9a18cc2b531422e205ab, NAME => 'testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:28:08,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741921_1099 (size=63) 2024-12-05T13:28:08,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741921_1099 (size=63) 2024-12-05T13:28:08,037 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:08,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741921_1099 (size=63) 2024-12-05T13:28:08,038 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,040 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName a 2024-12-05T13:28:08,040 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:08,040 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:08,041 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,042 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName b 2024-12-05T13:28:08,042 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:08,043 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:08,043 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,044 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName c 2024-12-05T13:28:08,044 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:08,045 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:08,045 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,046 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,047 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,047 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,047 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:08,049 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,051 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:28:08,052 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75e0611148ca9a18cc2b531422e205ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70310867, jitterRate=0.04771356284618378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:08,053 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75e0611148ca9a18cc2b531422e205ab: Writing region info on filesystem at 1733405288037Initializing all the Stores at 1733405288037Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405288038 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405288038Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405288038Cleaning up temporary data from old regions at 1733405288047 (+9 ms)Region opened successfully at 1733405288052 (+5 ms) 2024-12-05T13:28:08,053 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 75e0611148ca9a18cc2b531422e205ab, disabling compactions & flushes 2024-12-05T13:28:08,053 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:08,053 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:08,053 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. after waiting 0 ms 2024-12-05T13:28:08,053 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:08,053 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:08,053 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 75e0611148ca9a18cc2b531422e205ab: Waiting for close lock at 1733405288053Disabling compacts and flushes for region at 1733405288053Disabling writes for close at 1733405288053Writing region close event to WAL at 1733405288053Closed at 1733405288053 2024-12-05T13:28:08,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741920_1098 (size=93) 2024-12-05T13:28:08,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741920_1098 (size=93) 2024-12-05T13:28:08,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741920_1098 (size=93) 2024-12-05T13:28:08,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:28:08,061 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-64332669:(num 1733405288005) 2024-12-05T13:28:08,061 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:08,063 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:08,076 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063, exclude list is [], retry=0 2024-12-05T13:28:08,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:08,079 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:08,079 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:08,083 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 2024-12-05T13:28:08,083 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:28:08,083 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 75e0611148ca9a18cc2b531422e205ab, NAME => 'testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:08,084 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:08,084 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,084 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,085 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,086 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName a 2024-12-05T13:28:08,087 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:08,087 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:08,087 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,088 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName b 2024-12-05T13:28:08,088 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:08,089 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:08,089 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,090 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName c 2024-12-05T13:28:08,090 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:08,091 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:08,091 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,092 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,093 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,094 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,094 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,094 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:08,096 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:08,097 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75e0611148ca9a18cc2b531422e205ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64658063, jitterRate=-0.03651978075504303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:08,097 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75e0611148ca9a18cc2b531422e205ab: Writing region info on filesystem at 1733405288084Initializing all the Stores at 1733405288085 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405288085Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405288085Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405288085Cleaning up temporary data from old regions at 1733405288094 (+9 ms)Region opened successfully at 1733405288097 (+3 ms) 2024-12-05T13:28:08,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733405288103/Put/seqid=0 2024-12-05T13:28:08,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741923_1101 (size=4875) 2024-12-05T13:28:08,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741923_1101 (size=4875) 2024-12-05T13:28:08,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741923_1101 (size=4875) 2024-12-05T13:28:08,121 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733405288120/Put/seqid=0 2024-12-05T13:28:08,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741924_1102 (size=4875) 2024-12-05T13:28:08,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741924_1102 (size=4875) 2024-12-05T13:28:08,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741924_1102 (size=4875) 2024-12-05T13:28:08,135 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733405288135/Put/seqid=0 2024-12-05T13:28:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741925_1103 (size=4875) 2024-12-05T13:28:08,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741925_1103 (size=4875) 2024-12-05T13:28:08,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741925_1103 (size=4875) 2024-12-05T13:28:08,147 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 75e0611148ca9a18cc2b531422e205ab/a 2024-12-05T13:28:08,151 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-05T13:28:08,151 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-05T13:28:08,151 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 75e0611148ca9a18cc2b531422e205ab/a 2024-12-05T13:28:08,156 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-05T13:28:08,156 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-05T13:28:08,156 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 75e0611148ca9a18cc2b531422e205ab/a 2024-12-05T13:28:08,160 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-05T13:28:08,160 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-05T13:28:08,161 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 75e0611148ca9a18cc2b531422e205ab 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-05T13:28:08,178 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/.tmp/a/77e33df7073e43669922d229c3aaf814 is 55, key is testCompactedBulkLoadedFiles/a:a/1733405288098/Put/seqid=0 2024-12-05T13:28:08,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741926_1104 (size=5107) 2024-12-05T13:28:08,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741926_1104 (size=5107) 2024-12-05T13:28:08,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741926_1104 (size=5107) 2024-12-05T13:28:08,189 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/.tmp/a/77e33df7073e43669922d229c3aaf814 2024-12-05T13:28:08,195 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/.tmp/a/77e33df7073e43669922d229c3aaf814 as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814 2024-12-05T13:28:08,201 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814, entries=1, sequenceid=4, filesize=5.0 K 2024-12-05T13:28:08,203 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 75e0611148ca9a18cc2b531422e205ab in 42ms, sequenceid=4, compaction requested=false 2024-12-05T13:28:08,203 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 75e0611148ca9a18cc2b531422e205ab: 2024-12-05T13:28:08,204 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_ 2024-12-05T13:28:08,205 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_ 2024-12-05T13:28:08,206 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_ 2024-12-05T13:28:08,207 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile0 into 75e0611148ca9a18cc2b531422e205ab/a as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_ - updating store file list. 2024-12-05T13:28:08,212 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for dc837e0918f049ac8651a2d6f3615373_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-05T13:28:08,212 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_ into 75e0611148ca9a18cc2b531422e205ab/a 2024-12-05T13:28:08,212 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile0 into 75e0611148ca9a18cc2b531422e205ab/a (new location: hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_) 2024-12-05T13:28:08,214 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile1 into 75e0611148ca9a18cc2b531422e205ab/a as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_ - updating store file list. 2024-12-05T13:28:08,218 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-05T13:28:08,219 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_ into 75e0611148ca9a18cc2b531422e205ab/a 2024-12-05T13:28:08,219 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile1 into 75e0611148ca9a18cc2b531422e205ab/a (new location: hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_) 2024-12-05T13:28:08,220 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile2 into 75e0611148ca9a18cc2b531422e205ab/a as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_ - updating store file list. 2024-12-05T13:28:08,224 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-05T13:28:08,224 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_ into 75e0611148ca9a18cc2b531422e205ab/a 2024-12-05T13:28:08,224 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44203/hbase/testCompactedBulkLoadedFiles/hfile2 into 75e0611148ca9a18cc2b531422e205ab/a (new location: hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_) 2024-12-05T13:28:08,233 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T13:28:08,233 DEBUG [Time-limited test {}] regionserver.HStore(1541): 75e0611148ca9a18cc2b531422e205ab/a is initiating major compaction (all files) 2024-12-05T13:28:08,233 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 75e0611148ca9a18cc2b531422e205ab/a in testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:08,233 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_] into tmpdir=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/.tmp, totalSize=19.3 K 2024-12-05T13:28:08,234 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 77e33df7073e43669922d229c3aaf814, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733405288098 2024-12-05T13:28:08,235 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting dc837e0918f049ac8651a2d6f3615373_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-05T13:28:08,235 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-05T13:28:08,236 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-05T13:28:08,251 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/.tmp/a/8ebb14681d73441f971b8c2025ca4e91 is 55, key is testCompactedBulkLoadedFiles/a:a/1733405288098/Put/seqid=0 2024-12-05T13:28:08,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741927_1105 (size=6154) 2024-12-05T13:28:08,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741927_1105 (size=6154) 2024-12-05T13:28:08,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741927_1105 (size=6154) 2024-12-05T13:28:08,267 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/.tmp/a/8ebb14681d73441f971b8c2025ca4e91 as hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/8ebb14681d73441f971b8c2025ca4e91 2024-12-05T13:28:08,274 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 75e0611148ca9a18cc2b531422e205ab/a of 75e0611148ca9a18cc2b531422e205ab into 8ebb14681d73441f971b8c2025ca4e91(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T13:28:08,275 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 75e0611148ca9a18cc2b531422e205ab: 2024-12-05T13:28:08,275 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-05T13:28:08,275 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-05T13:28:08,308 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063, size=0 (0bytes) 2024-12-05T13:28:08,308 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 might be still open, length is 0 2024-12-05T13:28:08,308 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 2024-12-05T13:28:08,308 WARN [IPC Server handler 1 on default port 44203 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-05T13:28:08,309 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 after 1ms 2024-12-05T13:28:08,480 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:35932 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38677:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35932 dst: /127.0.0.1:38677 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38677 remote=/127.0.0.1:35932]. Total timeout mills is 60000, 59795 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:08,480 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:43626 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43626 dst: /127.0.0.1:43089 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:08,480 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33252 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:40037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33252 dst: /127.0.0.1:40037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:08,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741922_1106 (size=1659) 2024-12-05T13:28:08,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741922_1106 (size=1659) 2024-12-05T13:28:08,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741922_1106 (size=1659) 2024-12-05T13:28:12,309 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 after 4001ms 2024-12-05T13:28:12,312 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 took 4004ms 2024-12-05T13:28:12,315 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063; continuing. 2024-12-05T13:28:12,316 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 so closing down 2024-12-05T13:28:12,316 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:28:12,317 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733405288063.temp 2024-12-05T13:28:12,318 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000003-wal.1733405288063.temp 2024-12-05T13:28:12,319 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:28:12,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741928_1107 (size=698) 2024-12-05T13:28:12,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741928_1107 (size=698) 2024-12-05T13:28:12,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741928_1107 (size=698) 2024-12-05T13:28:12,326 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000003-wal.1733405288063.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-05T13:28:12,327 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000003-wal.1733405288063.temp to hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000008 2024-12-05T13:28:12,327 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063, size=0, length=0, corrupted=false, cancelled=false 2024-12-05T13:28:12,327 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063, journal: Splitting hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063, size=0 (0bytes) at 1733405288308Finishing writing output for hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 so closing down at 1733405292316 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000003-wal.1733405288063.temp at 1733405292318 (+2 ms)3 split writer threads finished at 1733405292319 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000003-wal.1733405288063.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733405292326 (+7 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000003-wal.1733405288063.temp to hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000008 at 1733405292327 (+1 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063, size=0, length=0, corrupted=false, cancelled=false at 1733405292327 2024-12-05T13:28:12,329 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405288063 2024-12-05T13:28:12,330 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000008 2024-12-05T13:28:12,330 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:12,332 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:12,345 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405292333, exclude list is [], retry=0 2024-12-05T13:28:12,347 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:12,348 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:12,348 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:12,349 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405292333 2024-12-05T13:28:12,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:28:12,350 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 75e0611148ca9a18cc2b531422e205ab, NAME => 'testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:12,350 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:12,350 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,350 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,351 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,352 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName a 2024-12-05T13:28:12,352 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,358 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814 2024-12-05T13:28:12,362 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/8ebb14681d73441f971b8c2025ca4e91 2024-12-05T13:28:12,366 DEBUG [StoreFileOpener-75e0611148ca9a18cc2b531422e205ab-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for dc837e0918f049ac8651a2d6f3615373_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-05T13:28:12,366 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_ 2024-12-05T13:28:12,371 DEBUG [StoreFileOpener-75e0611148ca9a18cc2b531422e205ab-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-05T13:28:12,371 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_ 2024-12-05T13:28:12,375 DEBUG [StoreFileOpener-75e0611148ca9a18cc2b531422e205ab-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-05T13:28:12,375 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_ 2024-12-05T13:28:12,375 WARN [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@f29d4d 2024-12-05T13:28:12,375 WARN [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@f29d4d 2024-12-05T13:28:12,375 WARN [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@f29d4d 2024-12-05T13:28:12,375 WARN [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@f29d4d 2024-12-05T13:28:12,375 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_] to archive 2024-12-05T13:28:12,376 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T13:28:12,378 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814 to hdfs://localhost:44203/hbase/archive/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/77e33df7073e43669922d229c3aaf814 2024-12-05T13:28:12,379 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_ to hdfs://localhost:44203/hbase/archive/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/dc837e0918f049ac8651a2d6f3615373_SeqId_4_ 2024-12-05T13:28:12,380 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_ to hdfs://localhost:44203/hbase/archive/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_ 2024-12-05T13:28:12,381 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_ to hdfs://localhost:44203/hbase/archive/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/a/ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_ 2024-12-05T13:28:12,381 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,381 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,382 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName b 2024-12-05T13:28:12,382 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,383 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,383 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,384 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75e0611148ca9a18cc2b531422e205ab columnFamilyName c 2024-12-05T13:28:12,384 DEBUG [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,384 INFO [StoreOpener-75e0611148ca9a18cc2b531422e205ab-1 {}] regionserver.HStore(327): Store=75e0611148ca9a18cc2b531422e205ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,384 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,386 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,386 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000008 2024-12-05T13:28:12,390 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 75e0611148ca9a18cc2b531422e205ab : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "75e0611148ca9a18cc2b531422e205ab" family_name: "a" compaction_input: "77e33df7073e43669922d229c3aaf814" compaction_input: "dc837e0918f049ac8651a2d6f3615373_SeqId_4_" compaction_input: "e55d8fce02f44fd9bd10b1cd3e546e13_SeqId_4_" compaction_input: "ec4c3b4f3cfd4b2880924525b72ff89c_SeqId_4_" compaction_output: "8ebb14681d73441f971b8c2025ca4e91" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-05T13:28:12,390 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-05T13:28:12,390 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000008 2024-12-05T13:28:12,391 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/0000000000000000008 2024-12-05T13:28:12,392 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,392 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,393 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:12,394 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75e0611148ca9a18cc2b531422e205ab 2024-12-05T13:28:12,396 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testCompactedBulkLoadedFiles/75e0611148ca9a18cc2b531422e205ab/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-05T13:28:12,397 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75e0611148ca9a18cc2b531422e205ab; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59566104, jitterRate=-0.11239588260650635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:12,397 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75e0611148ca9a18cc2b531422e205ab: Writing region info on filesystem at 1733405292350Initializing all the Stores at 1733405292351 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292351Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292351Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292351Cleaning up temporary data from old regions at 1733405292392 (+41 ms)Region opened successfully at 1733405292397 (+5 ms) 2024-12-05T13:28:12,400 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 75e0611148ca9a18cc2b531422e205ab, disabling compactions & flushes 2024-12-05T13:28:12,400 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:12,400 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:12,400 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. after waiting 0 ms 2024-12-05T13:28:12,400 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:12,401 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733405288000.75e0611148ca9a18cc2b531422e205ab. 2024-12-05T13:28:12,401 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 75e0611148ca9a18cc2b531422e205ab: Waiting for close lock at 1733405292400Disabling compacts and flushes for region at 1733405292400Disabling writes for close at 1733405292400Writing region close event to WAL at 1733405292401 (+1 ms)Closed at 1733405292401 2024-12-05T13:28:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741929_1108 (size=93) 2024-12-05T13:28:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741929_1108 (size=93) 2024-12-05T13:28:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741929_1108 (size=93) 2024-12-05T13:28:12,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:28:12,406 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733405292333) 2024-12-05T13:28:12,419 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=442 (was 428) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1864666616_22 at /127.0.0.1:35990 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44203 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:44203 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1864666616_22 at /127.0.0.1:43690 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1864666616_22 at /127.0.0.1:33356 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=242 (was 228) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=3898 (was 3982) 2024-12-05T13:28:12,419 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-05T13:28:12,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=242, ProcessCount=11, AvailableMemoryMB=3898 2024-12-05T13:28:12,436 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-05T13:28:12,450 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:28:12,452 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:28:12,452 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:28:12,455 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-87764618, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/hregion-87764618, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:12,469 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-87764618/hregion-87764618.1733405292455, exclude list is [], retry=0 2024-12-05T13:28:12,473 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:12,473 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:12,482 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:12,484 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-87764618/hregion-87764618.1733405292455 2024-12-05T13:28:12,489 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35457:35457),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953)] 2024-12-05T13:28:12,490 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4b5294c4188d1ee1ab9d0c81fb082137, NAME => 'testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44203/hbase 2024-12-05T13:28:12,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741931_1110 (size=67) 2024-12-05T13:28:12,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741931_1110 (size=67) 2024-12-05T13:28:12,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741931_1110 (size=67) 2024-12-05T13:28:12,501 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:12,503 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,504 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName a 2024-12-05T13:28:12,505 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,505 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,505 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,507 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName b 2024-12-05T13:28:12,507 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,507 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,507 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,509 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName c 2024-12-05T13:28:12,509 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,509 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,509 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,510 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,510 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,511 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,511 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,512 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:12,512 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,514 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:28:12,515 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b5294c4188d1ee1ab9d0c81fb082137; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74563633, jitterRate=0.1110847145318985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b5294c4188d1ee1ab9d0c81fb082137: Writing region info on filesystem at 1733405292502Initializing all the Stores at 1733405292502Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292502Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292503 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292503Cleaning up temporary data from old regions at 1733405292511 (+8 ms)Region opened successfully at 1733405292515 (+4 ms) 2024-12-05T13:28:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b5294c4188d1ee1ab9d0c81fb082137, disabling compactions & flushes 2024-12-05T13:28:12,515 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. after waiting 0 ms 2024-12-05T13:28:12,515 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,516 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,516 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b5294c4188d1ee1ab9d0c81fb082137: Waiting for close lock at 1733405292515Disabling compacts and flushes for region at 1733405292515Disabling writes for close at 1733405292515Writing region close event to WAL at 1733405292516 (+1 ms)Closed at 1733405292516 2024-12-05T13:28:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741930_1109 (size=93) 2024-12-05T13:28:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741930_1109 (size=93) 2024-12-05T13:28:12,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741930_1109 (size=93) 2024-12-05T13:28:12,522 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:28:12,522 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-87764618:(num 1733405292455) 2024-12-05T13:28:12,523 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:12,524 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:12,544 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525, exclude list is [], retry=0 2024-12-05T13:28:12,547 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:12,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:12,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:12,550 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525 2024-12-05T13:28:12,552 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:28:12,552 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b5294c4188d1ee1ab9d0c81fb082137, NAME => 'testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:12,552 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:12,553 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,553 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,555 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,556 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName a 2024-12-05T13:28:12,556 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,557 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,557 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,558 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName b 2024-12-05T13:28:12,558 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,559 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,559 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,560 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName c 2024-12-05T13:28:12,560 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,561 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,561 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T13:28:12,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,564 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,564 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,565 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:12,566 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,567 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b5294c4188d1ee1ab9d0c81fb082137; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69023476, jitterRate=0.02852994203567505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:12,568 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b5294c4188d1ee1ab9d0c81fb082137: Writing region info on filesystem at 1733405292553Initializing all the Stores at 1733405292554 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292554Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292554Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292554Cleaning up temporary data from old regions at 1733405292564 (+10 ms)Region opened successfully at 1733405292567 (+3 ms) 2024-12-05T13:28:12,590 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b5294c4188d1ee1ab9d0c81fb082137 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-05T13:28:12,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/a/1fe4a4de44b04d919e5ae830259f3f31 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733405292568/Put/seqid=0 2024-12-05T13:28:12,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741933_1112 (size=5958) 2024-12-05T13:28:12,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741933_1112 (size=5958) 2024-12-05T13:28:12,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741933_1112 (size=5958) 2024-12-05T13:28:12,620 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/a/1fe4a4de44b04d919e5ae830259f3f31 2024-12-05T13:28:12,627 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/a/1fe4a4de44b04d919e5ae830259f3f31 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/a/1fe4a4de44b04d919e5ae830259f3f31 2024-12-05T13:28:12,634 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/a/1fe4a4de44b04d919e5ae830259f3f31, entries=10, sequenceid=13, filesize=5.8 K 2024-12-05T13:28:12,635 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 4b5294c4188d1ee1ab9d0c81fb082137 in 45ms, sequenceid=13, compaction requested=false 2024-12-05T13:28:12,635 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4b5294c4188d1ee1ab9d0c81fb082137: 2024-12-05T13:28:12,659 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b5294c4188d1ee1ab9d0c81fb082137, disabling compactions & flushes 2024-12-05T13:28:12,659 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,659 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,659 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. after waiting 0 ms 2024-12-05T13:28:12,659 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,660 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,660 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:12,660 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b5294c4188d1ee1ab9d0c81fb082137: Waiting for close lock at 1733405292658Disabling compacts and flushes for region at 1733405292658Disabling writes for close at 1733405292659 (+1 ms)Writing region close event to WAL at 1733405292660 (+1 ms)Closed at 1733405292660 2024-12-05T13:28:12,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741932_1111 (size=5869) 2024-12-05T13:28:12,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741932_1111 (size=5869) 2024-12-05T13:28:12,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741932_1111 (size=5869) 2024-12-05T13:28:12,683 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525, size=5.7 K (5869bytes) 2024-12-05T13:28:12,683 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525 2024-12-05T13:28:12,683 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525 after 0ms 2024-12-05T13:28:12,686 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525 took 3ms 2024-12-05T13:28:12,688 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525 so closing down 2024-12-05T13:28:12,689 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:28:12,690 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733405292525.temp 2024-12-05T13:28:12,691 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000003-wal.1733405292525.temp 2024-12-05T13:28:12,692 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:28:12,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741934_1113 (size=5223) 2024-12-05T13:28:12,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741934_1113 (size=5223) 2024-12-05T13:28:12,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741934_1113 (size=5223) 2024-12-05T13:28:12,708 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000003-wal.1733405292525.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-05T13:28:12,709 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000003-wal.1733405292525.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000035 2024-12-05T13:28:12,710 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-12-05T13:28:12,710 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525, journal: Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525, size=5.7 K (5869bytes) at 1733405292683Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525 so closing down at 1733405292688 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000003-wal.1733405292525.temp at 1733405292691 (+3 ms)3 split writer threads finished at 1733405292692 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000003-wal.1733405292525.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733405292708 (+16 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000003-wal.1733405292525.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000035 at 1733405292709 (+1 ms)Processed 32 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1733405292710 (+1 ms) 2024-12-05T13:28:12,712 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292525 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405292525 2024-12-05T13:28:12,713 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000035 2024-12-05T13:28:12,713 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:12,715 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:12,728 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715, exclude list is [], retry=0 2024-12-05T13:28:12,731 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:12,731 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:12,731 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:12,733 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 2024-12-05T13:28:12,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:28:12,733 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b5294c4188d1ee1ab9d0c81fb082137, NAME => 'testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:28:12,733 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:12,734 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,734 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,737 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,739 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName a 2024-12-05T13:28:12,739 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,745 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/a/1fe4a4de44b04d919e5ae830259f3f31 2024-12-05T13:28:12,745 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,745 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,747 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName b 2024-12-05T13:28:12,747 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,747 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,748 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,749 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName c 2024-12-05T13:28:12,749 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:12,749 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:12,750 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,750 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,752 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,752 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000035 2024-12-05T13:28:12,756 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000035 2024-12-05T13:28:12,757 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b5294c4188d1ee1ab9d0c81fb082137 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-05T13:28:12,773 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/b/558f1dbc61b44a7daf42c5ebb6ac3c6b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733405292635/Put/seqid=0 2024-12-05T13:28:12,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741936_1115 (size=5958) 2024-12-05T13:28:12,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741936_1115 (size=5958) 2024-12-05T13:28:12,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741936_1115 (size=5958) 2024-12-05T13:28:12,786 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/b/558f1dbc61b44a7daf42c5ebb6ac3c6b 2024-12-05T13:28:12,807 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/c/3b436711a921460886581b7237477421 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733405292644/Put/seqid=0 2024-12-05T13:28:12,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741937_1116 (size=5958) 2024-12-05T13:28:12,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741937_1116 (size=5958) 2024-12-05T13:28:12,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741937_1116 (size=5958) 2024-12-05T13:28:12,818 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/c/3b436711a921460886581b7237477421 2024-12-05T13:28:12,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/b/558f1dbc61b44a7daf42c5ebb6ac3c6b as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/b/558f1dbc61b44a7daf42c5ebb6ac3c6b 2024-12-05T13:28:12,832 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/b/558f1dbc61b44a7daf42c5ebb6ac3c6b, entries=10, sequenceid=35, filesize=5.8 K 2024-12-05T13:28:12,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/c/3b436711a921460886581b7237477421 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/c/3b436711a921460886581b7237477421 2024-12-05T13:28:12,841 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/c/3b436711a921460886581b7237477421, entries=10, sequenceid=35, filesize=5.8 K 2024-12-05T13:28:12,841 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4b5294c4188d1ee1ab9d0c81fb082137 in 85ms, sequenceid=35, compaction requested=false; wal=null 2024-12-05T13:28:12,842 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000035 2024-12-05T13:28:12,843 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,843 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,844 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:12,845 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:12,848 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-05T13:28:12,850 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b5294c4188d1ee1ab9d0c81fb082137; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68175906, jitterRate=0.01590016484260559}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:12,850 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b5294c4188d1ee1ab9d0c81fb082137: Writing region info on filesystem at 1733405292734Initializing all the Stores at 1733405292735 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292735Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292737 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405292737Obtaining lock to block concurrent updates at 1733405292757 (+20 ms)Preparing flush snapshotting stores in 4b5294c4188d1ee1ab9d0c81fb082137 at 1733405292757Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733405292757Flushing stores of testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. at 1733405292757Flushing 4b5294c4188d1ee1ab9d0c81fb082137/b: creating writer at 1733405292757Flushing 4b5294c4188d1ee1ab9d0c81fb082137/b: appending metadata at 1733405292772 (+15 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/b: closing flushed file at 1733405292772Flushing 4b5294c4188d1ee1ab9d0c81fb082137/c: creating writer at 1733405292792 (+20 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/c: appending metadata at 1733405292806 (+14 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/c: closing flushed file at 1733405292807 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50b85cfb: reopening flushed file at 1733405292824 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@837d676: reopening flushed file at 1733405292832 (+8 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 4b5294c4188d1ee1ab9d0c81fb082137 in 85ms, sequenceid=35, compaction requested=false; wal=null at 1733405292841 (+9 ms)Cleaning up temporary data from old regions at 1733405292843 (+2 ms)Region opened successfully at 1733405292850 (+7 ms) 2024-12-05T13:28:12,930 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715, size=0 (0bytes) 2024-12-05T13:28:12,930 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 might be still open, length is 0 2024-12-05T13:28:12,930 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 2024-12-05T13:28:12,931 WARN [IPC Server handler 4 on default port 44203 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-05T13:28:12,931 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 after 1ms 2024-12-05T13:28:14,477 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:43738 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43738 dst: /127.0.0.1:43089 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43089 remote=/127.0.0.1:43738]. Total timeout mills is 60000, 58407 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:14,477 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:36064 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38677:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36064 dst: /127.0.0.1:38677 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:14,478 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_796864569_22 at /127.0.0.1:33400 [Receiving block BP-1492241733-172.17.0.3-1733405252624:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:40037:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33400 dst: /127.0.0.1:40037 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741935_1117 (size=5215) 2024-12-05T13:28:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741935_1117 (size=5215) 2024-12-05T13:28:14,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741935_1117 (size=5215) 2024-12-05T13:28:16,669 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-05T13:28:16,669 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-05T13:28:16,670 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-05T13:28:16,670 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-05T13:28:16,932 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 after 4002ms 2024-12-05T13:28:16,938 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 took 4008ms 2024-12-05T13:28:16,942 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715; continuing. 2024-12-05T13:28:16,942 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 so closing down 2024-12-05T13:28:16,942 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:28:16,943 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733405292715.temp 2024-12-05T13:28:16,945 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000037-wal.1733405292715.temp 2024-12-05T13:28:16,945 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:28:16,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741938_1118 (size=5223) 2024-12-05T13:28:16,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741938_1118 (size=5223) 2024-12-05T13:28:16,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741938_1118 (size=5223) 2024-12-05T13:28:16,953 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000037-wal.1733405292715.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-05T13:28:16,954 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000037-wal.1733405292715.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000066 2024-12-05T13:28:16,954 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715, size=0, length=0, corrupted=false, cancelled=false 2024-12-05T13:28:16,954 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715, journal: Splitting hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715, size=0 (0bytes) at 1733405292930Finishing writing output for hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 so closing down at 1733405296942 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000037-wal.1733405292715.temp at 1733405296945 (+3 ms)3 split writer threads finished at 1733405296945Closed recovered edits writer path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000037-wal.1733405292715.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733405296953 (+8 ms)Rename recovered edits hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000037-wal.1733405292715.temp to hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000066 at 1733405296954 (+1 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715, size=0, length=0, corrupted=false, cancelled=false at 1733405296954 2024-12-05T13:28:16,956 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 to hdfs://localhost:44203/hbase/oldWALs/wal.1733405292715 2024-12-05T13:28:16,957 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000066 2024-12-05T13:28:16,957 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-05T13:28:16,959 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44203/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449, archiveDir=hdfs://localhost:44203/hbase/oldWALs, maxLogs=32 2024-12-05T13:28:16,970 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405296959, exclude list is [], retry=0 2024-12-05T13:28:16,974 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40037,DS-5fe20c54-5be8-4e68-a457-25143a1f9317,DISK] 2024-12-05T13:28:16,974 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43089,DS-5be99c92-5113-4350-9597-c1ed3fbf9853,DISK] 2024-12-05T13:28:16,974 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38677,DS-153a313f-f6ef-473d-84f3-deb45a40d480,DISK] 2024-12-05T13:28:16,976 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405296959 2024-12-05T13:28:16,977 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32953:32953),(127.0.0.1/127.0.0.1:38653:38653),(127.0.0.1/127.0.0.1:35457:35457)] 2024-12-05T13:28:16,977 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:28:16,979 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:16,980 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName a 2024-12-05T13:28:16,980 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:16,985 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/a/1fe4a4de44b04d919e5ae830259f3f31 2024-12-05T13:28:16,986 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:16,986 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:16,987 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName b 2024-12-05T13:28:16,987 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:16,993 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/b/558f1dbc61b44a7daf42c5ebb6ac3c6b 2024-12-05T13:28:16,993 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:16,993 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:16,994 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b5294c4188d1ee1ab9d0c81fb082137 columnFamilyName c 2024-12-05T13:28:16,994 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:28:16,999 DEBUG [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/c/3b436711a921460886581b7237477421 2024-12-05T13:28:17,000 INFO [StoreOpener-4b5294c4188d1ee1ab9d0c81fb082137-1 {}] regionserver.HStore(327): Store=4b5294c4188d1ee1ab9d0c81fb082137/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:28:17,000 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:17,001 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:17,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:17,003 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000066 2024-12-05T13:28:17,008 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000066 2024-12-05T13:28:17,008 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b5294c4188d1ee1ab9d0c81fb082137 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-05T13:28:17,023 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/a/521838896d744abfa4c4753ae3e4b096 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733405292857/Put/seqid=0 2024-12-05T13:28:17,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741940_1120 (size=5958) 2024-12-05T13:28:17,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741940_1120 (size=5958) 2024-12-05T13:28:17,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741940_1120 (size=5958) 2024-12-05T13:28:17,031 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/a/521838896d744abfa4c4753ae3e4b096 2024-12-05T13:28:17,055 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/b/68727217638548d387d3b183b889d8cd is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733405292867/Put/seqid=0 2024-12-05T13:28:17,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741941_1121 (size=5958) 2024-12-05T13:28:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741941_1121 (size=5958) 2024-12-05T13:28:17,062 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/b/68727217638548d387d3b183b889d8cd 2024-12-05T13:28:17,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741941_1121 (size=5958) 2024-12-05T13:28:17,081 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/c/debcaf936cd74184902830af4adc0291 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733405292875/Put/seqid=0 2024-12-05T13:28:17,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741942_1122 (size=5958) 2024-12-05T13:28:17,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741942_1122 (size=5958) 2024-12-05T13:28:17,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741942_1122 (size=5958) 2024-12-05T13:28:17,089 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/c/debcaf936cd74184902830af4adc0291 2024-12-05T13:28:17,094 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/a/521838896d744abfa4c4753ae3e4b096 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/a/521838896d744abfa4c4753ae3e4b096 2024-12-05T13:28:17,099 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/a/521838896d744abfa4c4753ae3e4b096, entries=10, sequenceid=66, filesize=5.8 K 2024-12-05T13:28:17,100 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/b/68727217638548d387d3b183b889d8cd as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/b/68727217638548d387d3b183b889d8cd 2024-12-05T13:28:17,106 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/b/68727217638548d387d3b183b889d8cd, entries=10, sequenceid=66, filesize=5.8 K 2024-12-05T13:28:17,107 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/.tmp/c/debcaf936cd74184902830af4adc0291 as hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/c/debcaf936cd74184902830af4adc0291 2024-12-05T13:28:17,113 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/c/debcaf936cd74184902830af4adc0291, entries=10, sequenceid=66, filesize=5.8 K 2024-12-05T13:28:17,114 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4b5294c4188d1ee1ab9d0c81fb082137 in 106ms, sequenceid=66, compaction requested=false; wal=null 2024-12-05T13:28:17,115 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/0000000000000000066 2024-12-05T13:28:17,116 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:17,116 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:17,116 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T13:28:17,118 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b5294c4188d1ee1ab9d0c81fb082137 2024-12-05T13:28:17,120 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/hbase/data/default/testReplayEditsWrittenViaHRegion/4b5294c4188d1ee1ab9d0c81fb082137/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-05T13:28:17,121 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b5294c4188d1ee1ab9d0c81fb082137; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71572725, jitterRate=0.06651671230792999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T13:28:17,121 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b5294c4188d1ee1ab9d0c81fb082137: Writing region info on filesystem at 1733405296977Initializing all the Stores at 1733405296978 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405296978Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405296979 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733405296979Obtaining lock to block concurrent updates at 1733405297008 (+29 ms)Preparing flush snapshotting stores in 4b5294c4188d1ee1ab9d0c81fb082137 at 1733405297008Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733405297008Flushing stores of testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. at 1733405297008Flushing 4b5294c4188d1ee1ab9d0c81fb082137/a: creating writer at 1733405297008Flushing 4b5294c4188d1ee1ab9d0c81fb082137/a: appending metadata at 1733405297022 (+14 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/a: closing flushed file at 1733405297022Flushing 4b5294c4188d1ee1ab9d0c81fb082137/b: creating writer at 1733405297035 (+13 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/b: appending metadata at 1733405297054 (+19 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/b: closing flushed file at 1733405297054Flushing 4b5294c4188d1ee1ab9d0c81fb082137/c: creating writer at 1733405297067 (+13 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/c: appending metadata at 1733405297081 (+14 ms)Flushing 4b5294c4188d1ee1ab9d0c81fb082137/c: closing flushed file at 1733405297081Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16852d5d: reopening flushed file at 1733405297093 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bcd1e05: reopening flushed file at 1733405297099 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59d24576: reopening flushed file at 1733405297107 (+8 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 4b5294c4188d1ee1ab9d0c81fb082137 in 106ms, sequenceid=66, compaction requested=false; wal=null at 1733405297114 (+7 ms)Cleaning up temporary data from old regions at 1733405297116 (+2 ms)Region opened successfully at 1733405297121 (+5 ms) 2024-12-05T13:28:17,134 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b5294c4188d1ee1ab9d0c81fb082137, disabling compactions & flushes 2024-12-05T13:28:17,134 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:17,134 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:17,134 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. after waiting 0 ms 2024-12-05T13:28:17,134 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:17,136 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733405292450.4b5294c4188d1ee1ab9d0c81fb082137. 2024-12-05T13:28:17,136 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b5294c4188d1ee1ab9d0c81fb082137: Waiting for close lock at 1733405297134Disabling compacts and flushes for region at 1733405297134Disabling writes for close at 1733405297134Writing region close event to WAL at 1733405297136 (+2 ms)Closed at 1733405297136 2024-12-05T13:28:17,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741939_1119 (size=93) 2024-12-05T13:28:17,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741939_1119 (size=93) 2024-12-05T13:28:17,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741939_1119 (size=93) 2024-12-05T13:28:17,140 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-05T13:28:17,140 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733405296959) 2024-12-05T13:28:17,154 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=447 (was 442) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-541904192_22 at /127.0.0.1:34958 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44203 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-541904192_22 at /127.0.0.1:54640 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-541904192_22 at /127.0.0.1:43898 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (865318088) connection to localhost/127.0.0.1:44203 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 242), ProcessCount=11 (was 11), AvailableMemoryMB=4349 (was 3898) - AvailableMemoryMB LEAK? - 2024-12-05T13:28:17,154 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-12-05T13:28:17,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T13:28:17,154 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T13:28:17,155 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:28:17,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:28:17,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:28:17,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T13:28:17,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T13:28:17,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=83871044, stopped=false 2024-12-05T13:28:17,156 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=88aaa70743e7,43665,1733405256332 2024-12-05T13:28:17,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T13:28:17,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T13:28:17,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T13:28:17,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:28:17,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:28:17,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:28:17,195 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:28:17,196 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:28:17,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:28:17,197 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T13:28:17,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:28:17,197 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:28:17,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:28:17,198 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88aaa70743e7,33089,1733405257168' ***** 2024-12-05T13:28:17,198 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T13:28:17,198 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88aaa70743e7,44873,1733405257343' ***** 2024-12-05T13:28:17,198 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T13:28:17,199 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T13:28:17,199 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T13:28:17,199 INFO [RS:2;88aaa70743e7:44873 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T13:28:17,199 INFO [RS:0;88aaa70743e7:33089 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T13:28:17,199 INFO [RS:2;88aaa70743e7:44873 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T13:28:17,199 INFO [RS:0;88aaa70743e7:33089 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T13:28:17,199 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(959): stopping server 88aaa70743e7,33089,1733405257168 2024-12-05T13:28:17,199 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(3091): Received CLOSE for d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:28:17,199 INFO [RS:0;88aaa70743e7:33089 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:28:17,199 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T13:28:17,199 INFO [RS:0;88aaa70743e7:33089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;88aaa70743e7:33089. 2024-12-05T13:28:17,199 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T13:28:17,200 DEBUG [RS:0;88aaa70743e7:33089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:28:17,200 DEBUG [RS:0;88aaa70743e7:33089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:28:17,200 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(959): stopping server 88aaa70743e7,44873,1733405257343 2024-12-05T13:28:17,200 INFO [RS:2;88aaa70743e7:44873 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:28:17,200 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(976): stopping server 88aaa70743e7,33089,1733405257168; all regions closed. 2024-12-05T13:28:17,200 INFO [RS:2;88aaa70743e7:44873 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;88aaa70743e7:44873. 2024-12-05T13:28:17,200 DEBUG [RS:2;88aaa70743e7:44873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:28:17,200 DEBUG [RS:2;88aaa70743e7:44873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:28:17,200 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2889c120f533ce016b2cbcb6b393a73, disabling compactions & flushes 2024-12-05T13:28:17,200 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:28:17,200 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T13:28:17,200 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T13:28:17,200 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T13:28:17,200 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:28:17,200 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. after waiting 0 ms 2024-12-05T13:28:17,200 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T13:28:17,200 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:28:17,201 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T13:28:17,201 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d2889c120f533ce016b2cbcb6b393a73=testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73.} 2024-12-05T13:28:17,201 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T13:28:17,201 DEBUG [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d2889c120f533ce016b2cbcb6b393a73 2024-12-05T13:28:17,201 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T13:28:17,201 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T13:28:17,201 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T13:28:17,201 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T13:28:17,201 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-05T13:28:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741835_1011 (size=2598) 2024-12-05T13:28:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741835_1011 (size=2598) 2024-12-05T13:28:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741835_1011 (size=2598) 2024-12-05T13:28:17,207 DEBUG [RS:0;88aaa70743e7:33089 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88aaa70743e7%2C33089%2C1733405257168:(num 1733405258866) 2024-12-05T13:28:17,207 DEBUG [RS:0;88aaa70743e7:33089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] hbase.ChoreService(370): Chore service for: regionserver/88aaa70743e7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T13:28:17,207 INFO [regionserver/88aaa70743e7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T13:28:17,207 INFO [RS:0;88aaa70743e7:33089 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33089 2024-12-05T13:28:17,212 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/info/ab2ed83deda446b290d4201768ed5876 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73./info:regioninfo/1733405277183/Put/seqid=0 2024-12-05T13:28:17,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T13:28:17,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88aaa70743e7,33089,1733405257168 2024-12-05T13:28:17,215 INFO [RS:0;88aaa70743e7:33089 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:28:17,219 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/d2889c120f533ce016b2cbcb6b393a73/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-05T13:28:17,220 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:28:17,220 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2889c120f533ce016b2cbcb6b393a73: Waiting for close lock at 1733405297200Running coprocessor pre-close hooks at 1733405297200Disabling compacts and flushes for region at 1733405297200Disabling writes for close at 1733405297200Writing region close event to WAL at 1733405297211 (+11 ms)Running coprocessor post-close hooks at 1733405297220 (+9 ms)Closed at 1733405297220 2024-12-05T13:28:17,221 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733405273059.d2889c120f533ce016b2cbcb6b393a73. 2024-12-05T13:28:17,224 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88aaa70743e7,33089,1733405257168] 2024-12-05T13:28:17,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741943_1123 (size=6778) 2024-12-05T13:28:17,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741943_1123 (size=6778) 2024-12-05T13:28:17,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741943_1123 (size=6778) 2024-12-05T13:28:17,231 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/info/ab2ed83deda446b290d4201768ed5876 2024-12-05T13:28:17,232 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88aaa70743e7,33089,1733405257168 already deleted, retry=false 2024-12-05T13:28:17,232 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88aaa70743e7,33089,1733405257168 expired; onlineServers=1 2024-12-05T13:28:17,240 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/.tmp/info/ab2ed83deda446b290d4201768ed5876 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/info/ab2ed83deda446b290d4201768ed5876 2024-12-05T13:28:17,246 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/info/ab2ed83deda446b290d4201768ed5876, entries=8, sequenceid=23, filesize=6.6 K 2024-12-05T13:28:17,247 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 46ms, sequenceid=23, compaction requested=false 2024-12-05T13:28:17,247 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T13:28:17,252 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-05T13:28:17,253 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T13:28:17,253 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T13:28:17,253 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733405297201Running coprocessor pre-close hooks at 1733405297201Disabling compacts and flushes for region at 1733405297201Disabling writes for close at 1733405297201Obtaining lock to block concurrent updates at 1733405297201Preparing flush snapshotting stores in 1588230740 at 1733405297201Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733405297201Flushing stores of hbase:meta,,1.1588230740 at 1733405297202 (+1 ms)Flushing 1588230740/info: creating writer at 1733405297202Flushing 1588230740/info: appending metadata at 1733405297212 (+10 ms)Flushing 1588230740/info: closing flushed file at 1733405297212Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a8eff31: reopening flushed file at 1733405297239 (+27 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 46ms, sequenceid=23, compaction requested=false at 1733405297247 (+8 ms)Writing region close event to WAL at 1733405297248 (+1 ms)Running coprocessor post-close hooks at 1733405297253 (+5 ms)Closed at 1733405297253 2024-12-05T13:28:17,253 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T13:28:17,275 INFO [regionserver/88aaa70743e7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:28:17,275 INFO [regionserver/88aaa70743e7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:28:17,324 INFO [RS:0;88aaa70743e7:33089 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:28:17,324 INFO [RS:0;88aaa70743e7:33089 {}] regionserver.HRegionServer(1031): Exiting; stopping=88aaa70743e7,33089,1733405257168; zookeeper connection closed. 2024-12-05T13:28:17,324 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@31bd03ed {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@31bd03ed 2024-12-05T13:28:17,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:28:17,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33089-0x101a6daed470001, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:28:17,401 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(976): stopping server 88aaa70743e7,44873,1733405257343; all regions closed. 2024-12-05T13:28:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741893_1071 (size=2422) 2024-12-05T13:28:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741893_1071 (size=2422) 2024-12-05T13:28:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741893_1071 (size=2422) 2024-12-05T13:28:17,407 DEBUG [RS:2;88aaa70743e7:44873 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs 2024-12-05T13:28:17,407 INFO [RS:2;88aaa70743e7:44873 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88aaa70743e7%2C44873%2C1733405257343.meta:.meta(num 1733405276305) 2024-12-05T13:28:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741833_1009 (size=928) 2024-12-05T13:28:17,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741833_1009 (size=928) 2024-12-05T13:28:17,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741833_1009 (size=928) 2024-12-05T13:28:17,412 DEBUG [RS:2;88aaa70743e7:44873 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/oldWALs 2024-12-05T13:28:17,412 INFO [RS:2;88aaa70743e7:44873 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88aaa70743e7%2C44873%2C1733405257343:(num 1733405258866) 2024-12-05T13:28:17,412 DEBUG [RS:2;88aaa70743e7:44873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:28:17,412 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:28:17,412 INFO [RS:2;88aaa70743e7:44873 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T13:28:17,412 INFO [RS:2;88aaa70743e7:44873 {}] hbase.ChoreService(370): Chore service for: regionserver/88aaa70743e7:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T13:28:17,412 INFO [RS:2;88aaa70743e7:44873 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T13:28:17,412 INFO [regionserver/88aaa70743e7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T13:28:17,412 INFO [RS:2;88aaa70743e7:44873 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44873 2024-12-05T13:28:17,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88aaa70743e7,44873,1733405257343 2024-12-05T13:28:17,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T13:28:17,419 INFO [RS:2;88aaa70743e7:44873 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:28:17,427 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88aaa70743e7,44873,1733405257343] 2024-12-05T13:28:17,435 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88aaa70743e7,44873,1733405257343 already deleted, retry=false 2024-12-05T13:28:17,435 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88aaa70743e7,44873,1733405257343 expired; onlineServers=0 2024-12-05T13:28:17,436 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '88aaa70743e7,43665,1733405256332' ***** 2024-12-05T13:28:17,436 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T13:28:17,436 INFO [M:0;88aaa70743e7:43665 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:28:17,436 INFO [M:0;88aaa70743e7:43665 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T13:28:17,436 DEBUG [M:0;88aaa70743e7:43665 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T13:28:17,436 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T13:28:17,436 DEBUG [M:0;88aaa70743e7:43665 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T13:28:17,436 DEBUG [master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.small.0-1733405258532 {}] cleaner.HFileCleaner(306): Exit Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.small.0-1733405258532,5,FailOnTimeoutGroup] 2024-12-05T13:28:17,436 DEBUG [master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.large.0-1733405258531 {}] cleaner.HFileCleaner(306): Exit Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.large.0-1733405258531,5,FailOnTimeoutGroup] 2024-12-05T13:28:17,437 INFO [M:0;88aaa70743e7:43665 {}] hbase.ChoreService(370): Chore service for: master/88aaa70743e7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T13:28:17,437 INFO [M:0;88aaa70743e7:43665 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T13:28:17,437 DEBUG [M:0;88aaa70743e7:43665 {}] master.HMaster(1795): Stopping service threads 2024-12-05T13:28:17,437 INFO [M:0;88aaa70743e7:43665 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T13:28:17,437 INFO [M:0;88aaa70743e7:43665 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:28:17,438 INFO [M:0;88aaa70743e7:43665 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:28:17,438 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T13:28:17,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T13:28:17,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:28:17,444 DEBUG [M:0;88aaa70743e7:43665 {}] zookeeper.ZKUtil(347): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T13:28:17,444 WARN [M:0;88aaa70743e7:43665 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T13:28:17,445 INFO [M:0;88aaa70743e7:43665 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/.lastflushedseqids 2024-12-05T13:28:17,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741944_1124 (size=119) 2024-12-05T13:28:17,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741944_1124 (size=119) 2024-12-05T13:28:17,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741944_1124 (size=119) 2024-12-05T13:28:17,460 INFO [M:0;88aaa70743e7:43665 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:28:17,460 INFO [M:0;88aaa70743e7:43665 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T13:28:17,460 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T13:28:17,460 INFO [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:28:17,460 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:28:17,460 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T13:28:17,461 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:28:17,461 INFO [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-12-05T13:28:17,478 DEBUG [M:0;88aaa70743e7:43665 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/73f9134e77924ddca61cf76956506a8b is 82, key is hbase:meta,,1/info:regioninfo/1733405276483/Put/seqid=0 2024-12-05T13:28:17,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741945_1125 (size=6063) 2024-12-05T13:28:17,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741945_1125 (size=6063) 2024-12-05T13:28:17,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741945_1125 (size=6063) 2024-12-05T13:28:17,484 INFO [M:0;88aaa70743e7:43665 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/73f9134e77924ddca61cf76956506a8b 2024-12-05T13:28:17,504 DEBUG [M:0;88aaa70743e7:43665 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8daf825f645f4f259d270b54076dd9e9 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733405273504/Put/seqid=0 2024-12-05T13:28:17,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741946_1126 (size=7908) 2024-12-05T13:28:17,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741946_1126 (size=7908) 2024-12-05T13:28:17,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741946_1126 (size=7908) 2024-12-05T13:28:17,517 INFO [M:0;88aaa70743e7:43665 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8daf825f645f4f259d270b54076dd9e9 2024-12-05T13:28:17,522 INFO [M:0;88aaa70743e7:43665 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8daf825f645f4f259d270b54076dd9e9 2024-12-05T13:28:17,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:28:17,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44873-0x101a6daed470003, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:28:17,528 INFO [RS:2;88aaa70743e7:44873 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:28:17,528 INFO [RS:2;88aaa70743e7:44873 {}] regionserver.HRegionServer(1031): Exiting; stopping=88aaa70743e7,44873,1733405257343; zookeeper connection closed. 2024-12-05T13:28:17,528 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@74a805e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@74a805e3 2024-12-05T13:28:17,529 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-05T13:28:17,540 DEBUG [M:0;88aaa70743e7:43665 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9ac7b415515945aab9ef5da5a5a1cf07 is 69, key is 88aaa70743e7,33089,1733405257168/rs:state/1733405258569/Put/seqid=0 2024-12-05T13:28:17,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741947_1127 (size=5445) 2024-12-05T13:28:17,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741947_1127 (size=5445) 2024-12-05T13:28:17,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741947_1127 (size=5445) 2024-12-05T13:28:17,550 INFO [M:0;88aaa70743e7:43665 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9ac7b415515945aab9ef5da5a5a1cf07 2024-12-05T13:28:17,556 INFO [M:0;88aaa70743e7:43665 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9ac7b415515945aab9ef5da5a5a1cf07 2024-12-05T13:28:17,557 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/73f9134e77924ddca61cf76956506a8b as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/73f9134e77924ddca61cf76956506a8b 2024-12-05T13:28:17,562 INFO [M:0;88aaa70743e7:43665 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/73f9134e77924ddca61cf76956506a8b, entries=14, sequenceid=207, filesize=5.9 K 2024-12-05T13:28:17,563 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8daf825f645f4f259d270b54076dd9e9 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8daf825f645f4f259d270b54076dd9e9 2024-12-05T13:28:17,567 INFO [M:0;88aaa70743e7:43665 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8daf825f645f4f259d270b54076dd9e9 2024-12-05T13:28:17,567 INFO [M:0;88aaa70743e7:43665 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8daf825f645f4f259d270b54076dd9e9, entries=21, sequenceid=207, filesize=7.7 K 2024-12-05T13:28:17,568 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9ac7b415515945aab9ef5da5a5a1cf07 as hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9ac7b415515945aab9ef5da5a5a1cf07 2024-12-05T13:28:17,573 INFO [M:0;88aaa70743e7:43665 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9ac7b415515945aab9ef5da5a5a1cf07 2024-12-05T13:28:17,573 INFO [M:0;88aaa70743e7:43665 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/0fe93d7f-9879-f574-bf1a-f9a24eee068c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9ac7b415515945aab9ef5da5a5a1cf07, entries=3, sequenceid=207, filesize=5.3 K 2024-12-05T13:28:17,574 INFO [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=207, compaction requested=false 2024-12-05T13:28:17,579 INFO [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:28:17,579 DEBUG [M:0;88aaa70743e7:43665 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733405297460Disabling compacts and flushes for region at 1733405297460Disabling writes for close at 1733405297461 (+1 ms)Obtaining lock to block concurrent updates at 1733405297461Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733405297461Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1733405297461Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733405297462 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733405297462Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733405297477 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733405297477Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733405297489 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733405297503 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733405297503Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733405297522 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733405297539 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733405297539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36f579c8: reopening flushed file at 1733405297556 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48ba53b1: reopening flushed file at 1733405297562 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d03a9b: reopening flushed file at 1733405297567 (+5 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=207, compaction requested=false at 1733405297574 (+7 ms)Writing region close event to WAL at 1733405297579 (+5 ms)Closed at 1733405297579 2024-12-05T13:28:17,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38677 is added to blk_1073741830_1006 (size=99448) 2024-12-05T13:28:17,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40037 is added to blk_1073741830_1006 (size=99448) 2024-12-05T13:28:17,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43089 is added to blk_1073741830_1006 (size=99448) 2024-12-05T13:28:17,583 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T13:28:17,583 INFO [M:0;88aaa70743e7:43665 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T13:28:17,583 INFO [M:0;88aaa70743e7:43665 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43665 2024-12-05T13:28:17,584 INFO [M:0;88aaa70743e7:43665 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:28:17,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:28:17,694 INFO [M:0;88aaa70743e7:43665 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:28:17,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43665-0x101a6daed470000, quorum=127.0.0.1:61512, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:28:17,704 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405287079 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405287079 (inode 16688) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733405277843/wal.1733405287079 (inode 16688) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-05T13:28:17,708 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277698 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277698 (inode 16665) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733405277245/wal.1733405277698 (inode 16665) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-05T13:28:17,709 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733405268583/wal.1733405268695 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:17,712 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287845 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287845 (inode 16714) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733405287300/wal.1733405287845 (inode 16714) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-05T13:28:17,712 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733405287999/wal.1733405288063 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:17,713 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733405292449/wal.1733405292715 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:17,713 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733405261314/wal.1733405261406 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:28:17,715 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal.1733405261146 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal.1733405261146 (inode 16485) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733405260798/wal.1733405261146 (inode 16485) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-05T13:28:17,718 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal.1733405260493 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal.1733405260493 (inode 16462) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733405259777/wal.1733405260493 (inode 16462) Holder DFSClient_NONMAPREDUCE_796864569_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-05T13:28:17,753 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50b13206{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:28:17,757 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3267e685{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:28:17,757 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:28:17,757 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f8696d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:28:17,757 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37d4eea4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,STOPPED} 2024-12-05T13:28:17,761 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T13:28:17,761 WARN [BP-1492241733-172.17.0.3-1733405252624 heartbeating to localhost/127.0.0.1:44203 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T13:28:17,761 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T13:28:17,761 WARN [BP-1492241733-172.17.0.3-1733405252624 heartbeating to localhost/127.0.0.1:44203 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1492241733-172.17.0.3-1733405252624 (Datanode Uuid 8fff9606-6a5a-4fcd-ae2e-b4b7365386ef) service to localhost/127.0.0.1:44203 2024-12-05T13:28:17,762 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data5/current/BP-1492241733-172.17.0.3-1733405252624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:28:17,762 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data6/current/BP-1492241733-172.17.0.3-1733405252624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:28:17,763 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T13:28:17,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@494486cb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:28:17,765 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d076b20{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:28:17,765 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:28:17,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@152e2397{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:28:17,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fc499ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,STOPPED} 2024-12-05T13:28:17,766 WARN [BP-1492241733-172.17.0.3-1733405252624 heartbeating to localhost/127.0.0.1:44203 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T13:28:17,766 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T13:28:17,766 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T13:28:17,766 WARN [BP-1492241733-172.17.0.3-1733405252624 heartbeating to localhost/127.0.0.1:44203 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1492241733-172.17.0.3-1733405252624 (Datanode Uuid 26f974ac-76cd-4a28-932e-c73f194b6651) service to localhost/127.0.0.1:44203 2024-12-05T13:28:17,767 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data3/current/BP-1492241733-172.17.0.3-1733405252624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:28:17,767 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data4/current/BP-1492241733-172.17.0.3-1733405252624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:28:17,768 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T13:28:17,770 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5bb7bef3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:28:17,770 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4080a860{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:28:17,770 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:28:17,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c19a0d8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:28:17,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29f9ab09{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,STOPPED} 2024-12-05T13:28:17,772 WARN [BP-1492241733-172.17.0.3-1733405252624 heartbeating to localhost/127.0.0.1:44203 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T13:28:17,772 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T13:28:17,772 WARN [BP-1492241733-172.17.0.3-1733405252624 heartbeating to localhost/127.0.0.1:44203 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1492241733-172.17.0.3-1733405252624 (Datanode Uuid 392a5c38-1118-42f2-a5b7-5c852f57abb6) service to localhost/127.0.0.1:44203 2024-12-05T13:28:17,772 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T13:28:17,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data1/current/BP-1492241733-172.17.0.3-1733405252624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:28:17,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/cluster_0b340e61-4e1c-38b9-3cbb-b921d5d1999d/data/data2/current/BP-1492241733-172.17.0.3-1733405252624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:28:17,773 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T13:28:17,779 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T13:28:17,780 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:28:17,780 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:28:17,780 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:28:17,780 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9b82f8c6-914f-a506-6a25-adb41f57d30a/hadoop.log.dir/,STOPPED} 2024-12-05T13:28:17,789 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T13:28:17,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down