2024-11-23 15:46:06,343 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-23 15:46:06,353 main DEBUG Took 0.008687 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 15:46:06,354 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 15:46:06,354 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 15:46:06,355 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 15:46:06,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,362 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 15:46:06,373 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,375 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,376 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,377 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,378 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,379 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,380 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,380 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,381 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,382 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,382 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,383 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:46:06,384 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,384 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 15:46:06,386 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:46:06,387 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 15:46:06,388 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 15:46:06,389 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 15:46:06,390 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 15:46:06,390 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 15:46:06,398 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 15:46:06,400 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 15:46:06,402 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 15:46:06,402 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 15:46:06,403 main DEBUG createAppenders(={Console}) 2024-11-23 15:46:06,404 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-23 15:46:06,404 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-23 15:46:06,404 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-23 15:46:06,405 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 15:46:06,405 main DEBUG OutputStream closed 2024-11-23 15:46:06,405 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 15:46:06,405 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 15:46:06,406 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-23 15:46:06,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 15:46:06,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 15:46:06,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 15:46:06,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 15:46:06,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 15:46:06,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 15:46:06,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 15:46:06,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 15:46:06,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 15:46:06,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 15:46:06,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 15:46:06,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 15:46:06,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 15:46:06,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 15:46:06,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 15:46:06,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 15:46:06,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 15:46:06,481 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 15:46:06,483 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 15:46:06,483 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-23 15:46:06,484 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 15:46:06,484 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-23T15:46:06,696 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc 2024-11-23 15:46:06,700 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 15:46:06,700 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T15:46:06,708 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-23T15:46:06,716 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-23T15:46:06,737 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T15:46:06,778 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T15:46:06,778 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T15:46:06,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T15:46:06,803 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d, deleteOnExit=true 2024-11-23T15:46:06,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T15:46:06,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/test.cache.data in system properties and HBase conf 2024-11-23T15:46:06,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T15:46:06,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir in system properties and HBase conf 2024-11-23T15:46:06,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T15:46:06,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T15:46:06,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T15:46:06,877 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T15:46:06,965 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T15:46:06,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T15:46:06,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T15:46:06,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T15:46:06,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T15:46:06,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T15:46:06,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T15:46:06,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T15:46:06,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T15:46:06,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T15:46:06,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/nfs.dump.dir in system properties and HBase conf 2024-11-23T15:46:06,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/java.io.tmpdir in system properties and HBase conf 2024-11-23T15:46:06,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T15:46:06,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T15:46:06,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T15:46:07,963 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T15:46:08,028 INFO [Time-limited test {}] log.Log(170): Logging initialized @2378ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T15:46:08,093 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:46:08,146 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:46:08,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:46:08,166 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:46:08,167 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T15:46:08,186 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:46:08,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:46:08,192 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:46:08,372 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14f32243{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/java.io.tmpdir/jetty-localhost-37583-hadoop-hdfs-3_4_1-tests_jar-_-any-4701266308592065995/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T15:46:08,379 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:37583} 2024-11-23T15:46:08,380 INFO [Time-limited test {}] server.Server(415): Started @2730ms 2024-11-23T15:46:08,901 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:46:08,909 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:46:08,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:46:08,911 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:46:08,911 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T15:46:08,915 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:46:08,916 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:46:09,015 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d480c14{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/java.io.tmpdir/jetty-localhost-39831-hadoop-hdfs-3_4_1-tests_jar-_-any-1458454266082796621/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:46:09,016 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:39831} 2024-11-23T15:46:09,016 INFO [Time-limited test {}] server.Server(415): Started @3366ms 2024-11-23T15:46:09,062 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T15:46:09,170 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:46:09,178 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:46:09,195 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:46:09,196 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:46:09,196 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T15:46:09,197 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:46:09,197 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:46:09,323 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75d96791{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/java.io.tmpdir/jetty-localhost-35201-hadoop-hdfs-3_4_1-tests_jar-_-any-4402411307261460363/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:46:09,324 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:35201} 2024-11-23T15:46:09,324 INFO [Time-limited test {}] server.Server(415): Started @3674ms 2024-11-23T15:46:09,327 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T15:46:09,357 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:46:09,362 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:46:09,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:46:09,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:46:09,363 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-23T15:46:09,364 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:46:09,364 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:46:09,460 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@773b4a40{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/java.io.tmpdir/jetty-localhost-46751-hadoop-hdfs-3_4_1-tests_jar-_-any-12964809364813365306/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:46:09,461 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:46751} 2024-11-23T15:46:09,461 INFO [Time-limited test {}] server.Server(415): Started @3811ms 2024-11-23T15:46:09,463 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T15:46:10,516 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data3/current/BP-1981799117-172.17.0.2-1732376767483/current, will proceed with Du for space computation calculation, 2024-11-23T15:46:10,516 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data4/current/BP-1981799117-172.17.0.2-1732376767483/current, will proceed with Du for space computation calculation, 2024-11-23T15:46:10,516 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data1/current/BP-1981799117-172.17.0.2-1732376767483/current, will proceed with Du for space computation calculation, 2024-11-23T15:46:10,516 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data2/current/BP-1981799117-172.17.0.2-1732376767483/current, will proceed with Du for space computation calculation, 2024-11-23T15:46:10,531 WARN [Thread-133 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data6/current/BP-1981799117-172.17.0.2-1732376767483/current, will proceed with Du for space computation calculation, 2024-11-23T15:46:10,531 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data5/current/BP-1981799117-172.17.0.2-1732376767483/current, will proceed with Du for space computation calculation, 2024-11-23T15:46:10,554 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T15:46:10,554 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T15:46:10,558 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T15:46:10,599 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2fc3446a82faf6e7 with lease ID 0xc2c81a9860ce2265: Processing first storage report for DS-020d36d3-1e4b-42de-87db-071fd0edcd95 from datanode DatanodeRegistration(127.0.0.1:44917, datanodeUuid=586a75e0-c289-4b4c-be4c-3563ed9c80a5, infoPort=36223, infoSecurePort=0, ipcPort=45139, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483) 2024-11-23T15:46:10,600 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2fc3446a82faf6e7 with lease ID 0xc2c81a9860ce2265: from storage DS-020d36d3-1e4b-42de-87db-071fd0edcd95 node DatanodeRegistration(127.0.0.1:44917, datanodeUuid=586a75e0-c289-4b4c-be4c-3563ed9c80a5, infoPort=36223, infoSecurePort=0, ipcPort=45139, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T15:46:10,601 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafca82fcf439f63a with lease ID 0xc2c81a9860ce2264: Processing first storage report for DS-2a156afc-b4f3-409e-9d84-2bfc622042ac from datanode DatanodeRegistration(127.0.0.1:37735, datanodeUuid=ed3db639-a30a-4375-bc03-437a25bdabfc, infoPort=44785, infoSecurePort=0, ipcPort=46721, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483) 2024-11-23T15:46:10,601 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafca82fcf439f63a with lease ID 0xc2c81a9860ce2264: from storage DS-2a156afc-b4f3-409e-9d84-2bfc622042ac node DatanodeRegistration(127.0.0.1:37735, datanodeUuid=ed3db639-a30a-4375-bc03-437a25bdabfc, infoPort=44785, infoSecurePort=0, ipcPort=46721, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T15:46:10,601 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x633400afccc20e85 with lease ID 0xc2c81a9860ce2263: Processing first storage report for DS-b10ac704-f88d-415d-b341-90745ac2302f from datanode DatanodeRegistration(127.0.0.1:35471, datanodeUuid=8f49f37a-b0c1-4fe8-845f-47fc00815dc3, infoPort=37309, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483) 2024-11-23T15:46:10,602 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x633400afccc20e85 with lease ID 0xc2c81a9860ce2263: from storage DS-b10ac704-f88d-415d-b341-90745ac2302f node DatanodeRegistration(127.0.0.1:35471, datanodeUuid=8f49f37a-b0c1-4fe8-845f-47fc00815dc3, infoPort=37309, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T15:46:10,602 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2fc3446a82faf6e7 with lease ID 0xc2c81a9860ce2265: Processing first storage report for DS-e3421a87-a7c9-47c9-83c2-45186bcf7d6e from datanode DatanodeRegistration(127.0.0.1:44917, datanodeUuid=586a75e0-c289-4b4c-be4c-3563ed9c80a5, infoPort=36223, infoSecurePort=0, ipcPort=45139, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483) 2024-11-23T15:46:10,602 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2fc3446a82faf6e7 with lease ID 0xc2c81a9860ce2265: from storage DS-e3421a87-a7c9-47c9-83c2-45186bcf7d6e node DatanodeRegistration(127.0.0.1:44917, datanodeUuid=586a75e0-c289-4b4c-be4c-3563ed9c80a5, infoPort=36223, infoSecurePort=0, ipcPort=45139, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T15:46:10,602 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafca82fcf439f63a with lease ID 0xc2c81a9860ce2264: Processing first storage report for DS-1887b7c3-2e3f-4e68-8d2d-211f5ddf917b from datanode DatanodeRegistration(127.0.0.1:37735, datanodeUuid=ed3db639-a30a-4375-bc03-437a25bdabfc, infoPort=44785, infoSecurePort=0, ipcPort=46721, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483) 2024-11-23T15:46:10,602 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafca82fcf439f63a with lease ID 0xc2c81a9860ce2264: from storage DS-1887b7c3-2e3f-4e68-8d2d-211f5ddf917b node DatanodeRegistration(127.0.0.1:37735, datanodeUuid=ed3db639-a30a-4375-bc03-437a25bdabfc, infoPort=44785, infoSecurePort=0, ipcPort=46721, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-23T15:46:10,603 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x633400afccc20e85 with lease ID 0xc2c81a9860ce2263: Processing first storage report for DS-79e09f02-7637-45a2-bb78-1d8963af0e04 from datanode DatanodeRegistration(127.0.0.1:35471, datanodeUuid=8f49f37a-b0c1-4fe8-845f-47fc00815dc3, infoPort=37309, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483) 2024-11-23T15:46:10,603 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x633400afccc20e85 with lease ID 0xc2c81a9860ce2263: from storage DS-79e09f02-7637-45a2-bb78-1d8963af0e04 node DatanodeRegistration(127.0.0.1:35471, datanodeUuid=8f49f37a-b0c1-4fe8-845f-47fc00815dc3, infoPort=37309, infoSecurePort=0, ipcPort=43251, storageInfo=lv=-57;cid=testClusterID;nsid=1498695656;c=1732376767483), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T15:46:10,668 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc 2024-11-23T15:46:10,731 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/zookeeper_0, clientPort=63612, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T15:46:10,739 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63612 2024-11-23T15:46:10,749 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:10,752 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:10,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741825_1001 (size=7) 2024-11-23T15:46:10,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741825_1001 (size=7) 2024-11-23T15:46:10,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741825_1001 (size=7) 2024-11-23T15:46:11,367 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748 with version=8 2024-11-23T15:46:11,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/hbase-staging 2024-11-23T15:46:11,634 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b712f9af2c12:0 server-side Connection retries=45 2024-11-23T15:46:11,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:11,644 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:11,648 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:46:11,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:11,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:46:11,772 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T15:46:11,820 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T15:46:11,827 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T15:46:11,831 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:46:11,853 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24911 (auto-detected) 2024-11-23T15:46:11,854 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T15:46:11,871 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34021 2024-11-23T15:46:11,889 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34021 connecting to ZooKeeper ensemble=127.0.0.1:63612 2024-11-23T15:46:11,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:340210x0, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:46:12,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34021-0x10169a94b600000 connected 2024-11-23T15:46:12,092 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:46:12,113 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748, hbase.cluster.distributed=false 2024-11-23T15:46:12,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:46:12,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34021 2024-11-23T15:46:12,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34021 2024-11-23T15:46:12,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34021 2024-11-23T15:46:12,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34021 2024-11-23T15:46:12,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34021 2024-11-23T15:46:12,239 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b712f9af2c12:0 server-side Connection retries=45 2024-11-23T15:46:12,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,241 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:46:12,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:46:12,244 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T15:46:12,246 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:46:12,247 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45181 2024-11-23T15:46:12,249 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45181 connecting to ZooKeeper ensemble=127.0.0.1:63612 2024-11-23T15:46:12,250 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,253 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451810x0, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:46:12,275 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45181-0x10169a94b600001 connected 2024-11-23T15:46:12,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:46:12,279 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T15:46:12,286 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T15:46:12,289 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T15:46:12,294 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:46:12,297 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45181 2024-11-23T15:46:12,297 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45181 2024-11-23T15:46:12,297 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45181 2024-11-23T15:46:12,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45181 2024-11-23T15:46:12,298 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45181 2024-11-23T15:46:12,315 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b712f9af2c12:0 server-side Connection retries=45 2024-11-23T15:46:12,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,316 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:46:12,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:46:12,317 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T15:46:12,317 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:46:12,318 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34281 2024-11-23T15:46:12,320 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34281 connecting to ZooKeeper ensemble=127.0.0.1:63612 2024-11-23T15:46:12,320 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,323 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342810x0, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:46:12,358 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:342810x0, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:46:12,358 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34281-0x10169a94b600002 connected 2024-11-23T15:46:12,359 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T15:46:12,360 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T15:46:12,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T15:46:12,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:46:12,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34281 2024-11-23T15:46:12,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34281 2024-11-23T15:46:12,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34281 2024-11-23T15:46:12,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34281 2024-11-23T15:46:12,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34281 2024-11-23T15:46:12,381 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b712f9af2c12:0 server-side Connection retries=45 2024-11-23T15:46:12,381 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,381 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,381 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:46:12,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:46:12,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:46:12,382 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T15:46:12,382 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:46:12,384 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40435 2024-11-23T15:46:12,386 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40435 connecting to ZooKeeper ensemble=127.0.0.1:63612 2024-11-23T15:46:12,387 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404350x0, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:46:12,405 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404350x0, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:46:12,405 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40435-0x10169a94b600003 connected 2024-11-23T15:46:12,405 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T15:46:12,406 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T15:46:12,407 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T15:46:12,409 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:46:12,410 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40435 2024-11-23T15:46:12,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40435 2024-11-23T15:46:12,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40435 2024-11-23T15:46:12,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40435 2024-11-23T15:46:12,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40435 2024-11-23T15:46:12,435 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b712f9af2c12:34021 2024-11-23T15:46:12,436 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b712f9af2c12,34021,1732376771486 2024-11-23T15:46:12,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,451 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b712f9af2c12,34021,1732376771486 2024-11-23T15:46:12,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T15:46:12,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T15:46:12,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T15:46:12,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,483 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T15:46:12,485 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b712f9af2c12,34021,1732376771486 from backup master directory 2024-11-23T15:46:12,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b712f9af2c12,34021,1732376771486 2024-11-23T15:46:12,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:46:12,497 WARN [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:46:12,497 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b712f9af2c12,34021,1732376771486 2024-11-23T15:46:12,499 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T15:46:12,500 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T15:46:12,553 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/hbase.id] with ID: db9d882d-a2c9-4336-ae3a-b23aa9293c16 2024-11-23T15:46:12,553 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/.tmp/hbase.id 2024-11-23T15:46:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741826_1002 (size=42) 2024-11-23T15:46:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741826_1002 (size=42) 2024-11-23T15:46:12,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741826_1002 (size=42) 2024-11-23T15:46:12,570 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/.tmp/hbase.id]:[hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/hbase.id] 2024-11-23T15:46:12,618 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:46:12,623 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T15:46:12,641 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-23T15:46:12,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:12,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741827_1003 (size=196) 2024-11-23T15:46:12,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741827_1003 (size=196) 2024-11-23T15:46:12,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741827_1003 (size=196) 2024-11-23T15:46:12,693 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:46:12,695 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T15:46:12,701 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741828_1004 (size=1189) 2024-11-23T15:46:12,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741828_1004 (size=1189) 2024-11-23T15:46:12,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741828_1004 (size=1189) 2024-11-23T15:46:12,753 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store 2024-11-23T15:46:12,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741829_1005 (size=34) 2024-11-23T15:46:12,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741829_1005 (size=34) 2024-11-23T15:46:12,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741829_1005 (size=34) 2024-11-23T15:46:12,781 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T15:46:12,785 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:12,787 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T15:46:12,787 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:12,787 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:12,789 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T15:46:12,790 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:12,790 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:12,792 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732376772786Disabling compacts and flushes for region at 1732376772787 (+1 ms)Disabling writes for close at 1732376772790 (+3 ms)Writing region close event to WAL at 1732376772790Closed at 1732376772790 2024-11-23T15:46:12,794 WARN [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/.initializing 2024-11-23T15:46:12,794 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/WALs/b712f9af2c12,34021,1732376771486 2024-11-23T15:46:12,803 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:12,818 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C34021%2C1732376771486, suffix=, logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/WALs/b712f9af2c12,34021,1732376771486, archiveDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/oldWALs, maxLogs=10 2024-11-23T15:46:12,857 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/WALs/b712f9af2c12,34021,1732376771486/b712f9af2c12%2C34021%2C1732376771486.1732376772824, exclude list is [], retry=0 2024-11-23T15:46:12,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:12,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:12,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:12,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:12,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T15:46:12,919 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-23T15:46:12,977 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/WALs/b712f9af2c12,34021,1732376771486/b712f9af2c12%2C34021%2C1732376771486.1732376772824 2024-11-23T15:46:12,977 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:12,978 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:12,978 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:12,981 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:12,982 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T15:46:13,041 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:13,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T15:46:13,048 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:13,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T15:46:13,052 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:13,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T15:46:13,055 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:13,057 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,061 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,062 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,067 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,068 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,073 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:46:13,077 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:46:13,081 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:13,082 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64864697, jitterRate=-0.03344069421291351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:46:13,091 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732376772992Initializing all the Stores at 1732376772994 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376772995 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376772995Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376772996 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376772996Cleaning up temporary data from old regions at 1732376773068 (+72 ms)Region opened successfully at 1732376773090 (+22 ms) 2024-11-23T15:46:13,092 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T15:46:13,125 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d680db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:46:13,151 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T15:46:13,161 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T15:46:13,161 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T15:46:13,163 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T15:46:13,165 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T15:46:13,169 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T15:46:13,169 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T15:46:13,192 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T15:46:13,200 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T15:46:13,249 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T15:46:13,254 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T15:46:13,256 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T15:46:13,265 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T15:46:13,267 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T15:46:13,270 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T15:46:13,282 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T15:46:13,284 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T15:46:13,296 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T15:46:13,319 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T15:46:13,329 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,344 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b712f9af2c12,34021,1732376771486, sessionid=0x10169a94b600000, setting cluster-up flag (Was=false) 2024-11-23T15:46:13,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,398 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T15:46:13,403 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b712f9af2c12,34021,1732376771486 2024-11-23T15:46:13,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:13,448 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T15:46:13,450 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b712f9af2c12,34021,1732376771486 2024-11-23T15:46:13,458 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T15:46:13,520 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(746): ClusterId : db9d882d-a2c9-4336-ae3a-b23aa9293c16 2024-11-23T15:46:13,520 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(746): ClusterId : db9d882d-a2c9-4336-ae3a-b23aa9293c16 2024-11-23T15:46:13,520 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(746): ClusterId : db9d882d-a2c9-4336-ae3a-b23aa9293c16 2024-11-23T15:46:13,523 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T15:46:13,523 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T15:46:13,523 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T15:46:13,523 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T15:46:13,531 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T15:46:13,536 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T15:46:13,541 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b712f9af2c12,34021,1732376771486 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T15:46:13,567 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T15:46:13,567 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T15:46:13,567 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T15:46:13,567 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T15:46:13,567 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T15:46:13,567 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T15:46:13,568 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:46:13,568 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:46:13,568 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:46:13,569 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:46:13,569 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b712f9af2c12:0, corePoolSize=10, maxPoolSize=10 2024-11-23T15:46:13,569 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,569 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:46:13,569 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,572 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732376803572 2024-11-23T15:46:13,575 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T15:46:13,576 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:46:13,576 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T15:46:13,576 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T15:46:13,576 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T15:46:13,576 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T15:46:13,577 DEBUG [RS:2;b712f9af2c12:40435 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efde76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:46:13,579 DEBUG [RS:1;b712f9af2c12:34281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bfe5a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:46:13,580 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T15:46:13,580 DEBUG [RS:0;b712f9af2c12:45181 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d49595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:46:13,581 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T15:46:13,581 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T15:46:13,581 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T15:46:13,582 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T15:46:13,584 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,582 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,584 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T15:46:13,590 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T15:46:13,591 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T15:46:13,591 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T15:46:13,594 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T15:46:13,594 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b712f9af2c12:40435 2024-11-23T15:46:13,595 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T15:46:13,596 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b712f9af2c12:34281 2024-11-23T15:46:13,597 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732376773596,5,FailOnTimeoutGroup] 2024-11-23T15:46:13,597 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b712f9af2c12:45181 2024-11-23T15:46:13,598 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732376773597,5,FailOnTimeoutGroup] 2024-11-23T15:46:13,598 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,598 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T15:46:13,599 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T15:46:13,599 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T15:46:13,599 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T15:46:13,599 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,600 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T15:46:13,600 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,600 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T15:46:13,600 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T15:46:13,601 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T15:46:13,601 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T15:46:13,601 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T15:46:13,602 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(2659): reportForDuty to master=b712f9af2c12,34021,1732376771486 with port=34281, startcode=1732376772315 2024-11-23T15:46:13,602 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(2659): reportForDuty to master=b712f9af2c12,34021,1732376771486 with port=40435, startcode=1732376772380 2024-11-23T15:46:13,602 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(2659): reportForDuty to master=b712f9af2c12,34021,1732376771486 with port=45181, startcode=1732376772204 2024-11-23T15:46:13,614 DEBUG [RS:0;b712f9af2c12:45181 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T15:46:13,614 DEBUG [RS:1;b712f9af2c12:34281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T15:46:13,614 DEBUG [RS:2;b712f9af2c12:40435 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T15:46:13,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741831_1007 (size=1321) 2024-11-23T15:46:13,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741831_1007 (size=1321) 2024-11-23T15:46:13,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741831_1007 (size=1321) 2024-11-23T15:46:13,636 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T15:46:13,637 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748 2024-11-23T15:46:13,657 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50289, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T15:46:13,657 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T15:46:13,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741832_1008 (size=32) 2024-11-23T15:46:13,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741832_1008 (size=32) 2024-11-23T15:46:13,657 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43889, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T15:46:13,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741832_1008 (size=32) 2024-11-23T15:46:13,661 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:13,664 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34021 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b712f9af2c12,45181,1732376772204 2024-11-23T15:46:13,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T15:46:13,666 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34021 {}] master.ServerManager(517): Registering regionserver=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:13,667 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T15:46:13,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:13,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T15:46:13,672 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T15:46:13,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:13,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T15:46:13,676 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T15:46:13,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:13,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T15:46:13,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34021 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b712f9af2c12,34281,1732376772315 2024-11-23T15:46:13,679 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34021 {}] master.ServerManager(517): Registering regionserver=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:13,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T15:46:13,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:13,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:13,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T15:46:13,683 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748 2024-11-23T15:46:13,683 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33345 2024-11-23T15:46:13,683 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T15:46:13,684 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34021 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b712f9af2c12,40435,1732376772380 2024-11-23T15:46:13,684 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34021 {}] master.ServerManager(517): Registering regionserver=b712f9af2c12,40435,1732376772380 2024-11-23T15:46:13,684 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748 2024-11-23T15:46:13,684 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33345 2024-11-23T15:46:13,685 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T15:46:13,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740 2024-11-23T15:46:13,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740 2024-11-23T15:46:13,688 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748 2024-11-23T15:46:13,688 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33345 2024-11-23T15:46:13,688 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T15:46:13,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T15:46:13,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T15:46:13,690 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:46:13,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T15:46:13,696 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:13,697 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68952280, jitterRate=0.02746903896331787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:46:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732376773662Initializing all the Stores at 1732376773663 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376773664 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376773664Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376773664Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376773664Cleaning up temporary data from old regions at 1732376773689 (+25 ms)Region opened successfully at 1732376773700 (+11 ms) 2024-11-23T15:46:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T15:46:13,701 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T15:46:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T15:46:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T15:46:13,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T15:46:13,702 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T15:46:13,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732376773701Disabling compacts and flushes for region at 1732376773701Disabling writes for close at 1732376773701Writing region close event to WAL at 1732376773702 (+1 ms)Closed at 1732376773702 2024-11-23T15:46:13,705 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:46:13,705 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T15:46:13,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T15:46:13,719 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:46:13,722 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T15:46:13,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:46:13,748 DEBUG [RS:0;b712f9af2c12:45181 {}] zookeeper.ZKUtil(111): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b712f9af2c12,45181,1732376772204 2024-11-23T15:46:13,748 DEBUG [RS:1;b712f9af2c12:34281 {}] zookeeper.ZKUtil(111): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b712f9af2c12,34281,1732376772315 2024-11-23T15:46:13,749 WARN [RS:0;b712f9af2c12:45181 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:46:13,749 WARN [RS:1;b712f9af2c12:34281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:46:13,749 INFO [RS:0;b712f9af2c12:45181 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:13,749 INFO [RS:1;b712f9af2c12:34281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:13,749 DEBUG [RS:2;b712f9af2c12:40435 {}] zookeeper.ZKUtil(111): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b712f9af2c12,40435,1732376772380 2024-11-23T15:46:13,749 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204 2024-11-23T15:46:13,749 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,34281,1732376772315 2024-11-23T15:46:13,749 WARN [RS:2;b712f9af2c12:40435 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:46:13,749 INFO [RS:2;b712f9af2c12:40435 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:13,749 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380 2024-11-23T15:46:13,750 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b712f9af2c12,40435,1732376772380] 2024-11-23T15:46:13,750 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b712f9af2c12,45181,1732376772204] 2024-11-23T15:46:13,751 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b712f9af2c12,34281,1732376772315] 2024-11-23T15:46:13,774 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T15:46:13,774 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T15:46:13,774 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T15:46:13,790 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T15:46:13,790 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T15:46:13,790 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T15:46:13,796 INFO [RS:1;b712f9af2c12:34281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T15:46:13,796 INFO [RS:0;b712f9af2c12:45181 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T15:46:13,796 INFO [RS:2;b712f9af2c12:40435 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T15:46:13,796 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,796 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,796 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,798 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T15:46:13,798 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T15:46:13,798 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T15:46:13,804 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T15:46:13,804 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T15:46:13,804 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T15:46:13,806 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,806 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,806 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:46:13,807 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:46:13,807 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,807 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:46:13,808 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:0;b712f9af2c12:45181 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:46:13,808 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:46:13,808 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:46:13,808 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:46:13,808 DEBUG [RS:1;b712f9af2c12:34281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:46:13,808 DEBUG [RS:2;b712f9af2c12:40435 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:46:13,810 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,811 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,811 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,811 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,811 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,811 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34281,1732376772315-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,45181,1732376772204-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:46:13,812 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,813 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,40435,1732376772380-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:46:13,830 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T15:46:13,831 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T15:46:13,831 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T15:46:13,832 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,40435,1732376772380-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,832 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,45181,1732376772204-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,832 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34281,1732376772315-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,833 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,833 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,833 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,833 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.Replication(171): b712f9af2c12,45181,1732376772204 started 2024-11-23T15:46:13,833 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.Replication(171): b712f9af2c12,40435,1732376772380 started 2024-11-23T15:46:13,833 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.Replication(171): b712f9af2c12,34281,1732376772315 started 2024-11-23T15:46:13,850 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,851 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1482): Serving as b712f9af2c12,45181,1732376772204, RpcServer on b712f9af2c12/172.17.0.2:45181, sessionid=0x10169a94b600001 2024-11-23T15:46:13,851 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T15:46:13,851 DEBUG [RS:0;b712f9af2c12:45181 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b712f9af2c12,45181,1732376772204 2024-11-23T15:46:13,852 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,45181,1732376772204' 2024-11-23T15:46:13,852 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T15:46:13,853 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T15:46:13,853 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T15:46:13,853 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T15:46:13,853 DEBUG [RS:0;b712f9af2c12:45181 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b712f9af2c12,45181,1732376772204 2024-11-23T15:46:13,854 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,45181,1732376772204' 2024-11-23T15:46:13,854 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T15:46:13,854 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T15:46:13,855 DEBUG [RS:0;b712f9af2c12:45181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T15:46:13,855 INFO [RS:0;b712f9af2c12:45181 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T15:46:13,855 INFO [RS:0;b712f9af2c12:45181 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T15:46:13,857 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,857 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:13,857 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1482): Serving as b712f9af2c12,40435,1732376772380, RpcServer on b712f9af2c12/172.17.0.2:40435, sessionid=0x10169a94b600003 2024-11-23T15:46:13,857 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1482): Serving as b712f9af2c12,34281,1732376772315, RpcServer on b712f9af2c12/172.17.0.2:34281, sessionid=0x10169a94b600002 2024-11-23T15:46:13,857 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T15:46:13,857 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T15:46:13,857 DEBUG [RS:2;b712f9af2c12:40435 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b712f9af2c12,40435,1732376772380 2024-11-23T15:46:13,857 DEBUG [RS:1;b712f9af2c12:34281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b712f9af2c12,34281,1732376772315 2024-11-23T15:46:13,857 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,34281,1732376772315' 2024-11-23T15:46:13,857 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,40435,1732376772380' 2024-11-23T15:46:13,857 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T15:46:13,857 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T15:46:13,858 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T15:46:13,858 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T15:46:13,859 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T15:46:13,859 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T15:46:13,859 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T15:46:13,859 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T15:46:13,859 DEBUG [RS:1;b712f9af2c12:34281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b712f9af2c12,34281,1732376772315 2024-11-23T15:46:13,859 DEBUG [RS:2;b712f9af2c12:40435 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b712f9af2c12,40435,1732376772380 2024-11-23T15:46:13,859 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,34281,1732376772315' 2024-11-23T15:46:13,859 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,40435,1732376772380' 2024-11-23T15:46:13,859 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T15:46:13,859 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T15:46:13,859 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T15:46:13,859 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T15:46:13,860 DEBUG [RS:2;b712f9af2c12:40435 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T15:46:13,860 DEBUG [RS:1;b712f9af2c12:34281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T15:46:13,860 INFO [RS:2;b712f9af2c12:40435 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T15:46:13,860 INFO [RS:1;b712f9af2c12:34281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T15:46:13,860 INFO [RS:2;b712f9af2c12:40435 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T15:46:13,860 INFO [RS:1;b712f9af2c12:34281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T15:46:13,873 WARN [b712f9af2c12:34021 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T15:46:13,960 INFO [RS:0;b712f9af2c12:45181 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:13,961 INFO [RS:2;b712f9af2c12:40435 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:13,961 INFO [RS:1;b712f9af2c12:34281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:13,963 INFO [RS:0;b712f9af2c12:45181 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C45181%2C1732376772204, suffix=, logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204, archiveDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs, maxLogs=32 2024-11-23T15:46:13,963 INFO [RS:1;b712f9af2c12:34281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C34281%2C1732376772315, suffix=, logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,34281,1732376772315, archiveDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs, maxLogs=32 2024-11-23T15:46:13,964 INFO [RS:2;b712f9af2c12:40435 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C40435%2C1732376772380, suffix=, logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380, archiveDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs, maxLogs=32 2024-11-23T15:46:13,979 DEBUG [RS:0;b712f9af2c12:45181 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204/b712f9af2c12%2C45181%2C1732376772204.1732376773966, exclude list is [], retry=0 2024-11-23T15:46:13,981 DEBUG [RS:1;b712f9af2c12:34281 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,34281,1732376772315/b712f9af2c12%2C34281%2C1732376772315.1732376773966, exclude list is [], retry=0 2024-11-23T15:46:13,981 DEBUG [RS:2;b712f9af2c12:40435 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380/b712f9af2c12%2C40435%2C1732376772380.1732376773966, exclude list is [], retry=0 2024-11-23T15:46:13,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:13,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:13,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:13,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:13,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:13,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:13,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:13,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:13,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:14,030 INFO [RS:2;b712f9af2c12:40435 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380/b712f9af2c12%2C40435%2C1732376772380.1732376773966 2024-11-23T15:46:14,030 INFO [RS:0;b712f9af2c12:45181 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204/b712f9af2c12%2C45181%2C1732376772204.1732376773966 2024-11-23T15:46:14,030 DEBUG [RS:2;b712f9af2c12:40435 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:14,030 DEBUG [RS:0;b712f9af2c12:45181 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:14,031 INFO [RS:1;b712f9af2c12:34281 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,34281,1732376772315/b712f9af2c12%2C34281%2C1732376772315.1732376773966 2024-11-23T15:46:14,032 DEBUG [RS:1;b712f9af2c12:34281 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309)] 2024-11-23T15:46:14,128 DEBUG [b712f9af2c12:34021 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-23T15:46:14,141 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(204): Hosts are {b712f9af2c12=0} racks are {/default-rack=0} 2024-11-23T15:46:14,147 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T15:46:14,147 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T15:46:14,147 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T15:46:14,147 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T15:46:14,147 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T15:46:14,147 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T15:46:14,147 INFO [b712f9af2c12:34021 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T15:46:14,147 INFO [b712f9af2c12:34021 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T15:46:14,147 INFO [b712f9af2c12:34021 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T15:46:14,147 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T15:46:14,156 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:14,161 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,45181,1732376772204, state=OPENING 2024-11-23T15:46:14,204 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T15:46:14,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:14,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:14,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:14,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:14,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,220 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:46:14,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,45181,1732376772204}] 2024-11-23T15:46:14,402 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T15:46:14,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40099, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T15:46:14,419 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T15:46:14,419 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:14,420 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T15:46:14,423 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C45181%2C1732376772204.meta, suffix=.meta, logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204, archiveDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs, maxLogs=32 2024-11-23T15:46:14,436 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, exclude list is [], retry=0 2024-11-23T15:46:14,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:14,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:14,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:14,472 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta 2024-11-23T15:46:14,472 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:14,472 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:14,474 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T15:46:14,477 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T15:46:14,483 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T15:46:14,488 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T15:46:14,488 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:14,489 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T15:46:14,489 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T15:46:14,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T15:46:14,494 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T15:46:14,494 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:14,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:14,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T15:46:14,496 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T15:46:14,497 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:14,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:14,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T15:46:14,499 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T15:46:14,499 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:14,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:14,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T15:46:14,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T15:46:14,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:14,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:14,503 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T15:46:14,504 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740 2024-11-23T15:46:14,507 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740 2024-11-23T15:46:14,510 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T15:46:14,510 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T15:46:14,511 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:46:14,514 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T15:46:14,516 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67979963, jitterRate=0.0129803866147995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:46:14,516 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T15:46:14,517 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732376774489Writing region info on filesystem at 1732376774489Initializing all the Stores at 1732376774492 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376774492Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376774492Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376774492Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376774492Cleaning up temporary data from old regions at 1732376774510 (+18 ms)Running coprocessor post-open hooks at 1732376774516 (+6 ms)Region opened successfully at 1732376774517 (+1 ms) 2024-11-23T15:46:14,524 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732376774394 2024-11-23T15:46:14,534 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T15:46:14,535 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T15:46:14,537 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:14,539 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,45181,1732376772204, state=OPEN 2024-11-23T15:46:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:14,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:14,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:14,563 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:14,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T15:46:14,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,45181,1732376772204 in 341 msec 2024-11-23T15:46:14,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T15:46:14,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 864 msec 2024-11-23T15:46:14,581 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:46:14,581 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T15:46:14,601 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:14,602 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1] 2024-11-23T15:46:14,626 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:46:14,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52835, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:46:14,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1610 sec 2024-11-23T15:46:14,648 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732376774648, completionTime=-1 2024-11-23T15:46:14,651 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-23T15:46:14,651 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T15:46:14,674 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-23T15:46:14,675 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732376834675 2024-11-23T15:46:14,675 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732376894675 2024-11-23T15:46:14,675 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-23T15:46:14,679 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T15:46:14,686 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34021,1732376771486-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:14,686 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34021,1732376771486-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:14,686 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34021,1732376771486-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:14,687 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b712f9af2c12:34021, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:14,688 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:14,688 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:14,694 DEBUG [master/b712f9af2c12:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T15:46:14,714 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.216sec 2024-11-23T15:46:14,715 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T15:46:14,716 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T15:46:14,717 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T15:46:14,717 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T15:46:14,717 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T15:46:14,718 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34021,1732376771486-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:46:14,718 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34021,1732376771486-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T15:46:14,723 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T15:46:14,723 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T15:46:14,724 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,34021,1732376771486-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:14,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a02bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:46:14,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,34021,-1 for getting cluster id 2024-11-23T15:46:14,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:46:14,741 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'db9d882d-a2c9-4336-ae3a-b23aa9293c16' 2024-11-23T15:46:14,744 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:46:14,744 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "db9d882d-a2c9-4336-ae3a-b23aa9293c16" 2024-11-23T15:46:14,744 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b02c5b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:46:14,744 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,34021,-1] 2024-11-23T15:46:14,747 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:46:14,750 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:14,750 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:46:14,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c1cca64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:46:14,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:14,760 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1] 2024-11-23T15:46:14,761 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:46:14,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:46:14,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b712f9af2c12,34021,1732376771486 2024-11-23T15:46:14,788 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33345/hbase 2024-11-23T15:46:14,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=8195 2024-11-23T15:46:14,827 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:14,831 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:14,832 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:14,837 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98467641, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-98467641, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:14,854 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98467641/hregion-98467641.1732376774840, exclude list is [], retry=0 2024-11-23T15:46:14,858 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:14,858 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:14,859 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:14,887 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98467641/hregion-98467641.1732376774840 2024-11-23T15:46:14,888 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309)] 2024-11-23T15:46:14,888 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b8b089c015a29e496a5185fa76e2f939, NAME => 'testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:14,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741838_1014 (size=64) 2024-11-23T15:46:14,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741838_1014 (size=64) 2024-11-23T15:46:14,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741838_1014 (size=64) 2024-11-23T15:46:14,903 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:14,905 INFO [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:14,907 INFO [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b089c015a29e496a5185fa76e2f939 columnFamilyName a 2024-11-23T15:46:14,907 DEBUG [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:14,908 INFO [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] regionserver.HStore(327): Store=b8b089c015a29e496a5185fa76e2f939/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:14,908 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:14,909 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:14,910 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:14,911 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:14,911 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:14,913 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:14,917 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:14,918 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b8b089c015a29e496a5185fa76e2f939; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67671353, jitterRate=0.008381739258766174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b8b089c015a29e496a5185fa76e2f939: Writing region info on filesystem at 1732376774903Initializing all the Stores at 1732376774904 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376774904Cleaning up temporary data from old regions at 1732376774911 (+7 ms)Region opened successfully at 1732376774919 (+8 ms) 2024-11-23T15:46:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b8b089c015a29e496a5185fa76e2f939, disabling compactions & flushes 2024-11-23T15:46:14,919 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939. 2024-11-23T15:46:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939. 2024-11-23T15:46:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939. after waiting 0 ms 2024-11-23T15:46:14,919 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939. 2024-11-23T15:46:14,920 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939. 2024-11-23T15:46:14,920 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b8b089c015a29e496a5185fa76e2f939: Waiting for close lock at 1732376774919Disabling compacts and flushes for region at 1732376774919Disabling writes for close at 1732376774919Writing region close event to WAL at 1732376774920 (+1 ms)Closed at 1732376774920 2024-11-23T15:46:14,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741837_1013 (size=175) 2024-11-23T15:46:14,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741837_1013 (size=175) 2024-11-23T15:46:14,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741837_1013 (size=175) 2024-11-23T15:46:14,931 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:14,931 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98467641:(num 1732376774840) 2024-11-23T15:46:14,933 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T15:46:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741839_1015 (size=469) 2024-11-23T15:46:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741839_1015 (size=469) 2024-11-23T15:46:14,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741839_1015 (size=469) 2024-11-23T15:46:14,979 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T15:46:15,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741840_1016 (size=322) 2024-11-23T15:46:15,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741840_1016 (size=322) 2024-11-23T15:46:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741840_1016 (size=322) 2024-11-23T15:46:15,037 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1, size=469 (469bytes) 2024-11-23T15:46:15,038 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-23T15:46:15,038 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-23T15:46:15,038 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1 2024-11-23T15:46:15,043 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1 after 3ms 2024-11-23T15:46:15,073 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:15,074 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1 took 37ms 2024-11-23T15:46:15,078 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1 so closing down 2024-11-23T15:46:15,078 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:15,081 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-23T15:46:15,112 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000001-wal-1.temp 2024-11-23T15:46:15,113 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:15,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741841_1017 (size=469) 2024-11-23T15:46:15,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741841_1017 (size=469) 2024-11-23T15:46:15,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741841_1017 (size=469) 2024-11-23T15:46:15,125 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:15,128 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002 2024-11-23T15:46:15,133 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-23T15:46:15,134 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1, journal: Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1, size=469 (469bytes) at 1732376775037Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1 so closing down at 1732376775078 (+41 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000001-wal-1.temp at 1732376775112 (+34 ms)3 split writer threads finished at 1732376775113 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732376775125 (+12 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002 at 1732376775128 (+3 ms)Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732376775134 (+6 ms) 2024-11-23T15:46:15,154 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2, size=322 (322bytes) 2024-11-23T15:46:15,154 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2 2024-11-23T15:46:15,155 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2 after 1ms 2024-11-23T15:46:15,181 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:15,182 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2 took 28ms 2024-11-23T15:46:15,185 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2 so closing down 2024-11-23T15:46:15,185 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:15,187 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-23T15:46:15,216 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002-wal-2.temp 2024-11-23T15:46:15,216 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:15,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741842_1018 (size=322) 2024-11-23T15:46:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741842_1018 (size=322) 2024-11-23T15:46:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741842_1018 (size=322) 2024-11-23T15:46:15,226 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:15,254 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:15,257 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-23T15:46:15,259 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-23T15:46:15,259 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2, journal: Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2, size=322 (322bytes) at 1732376775154Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2 so closing down at 1732376775185 (+31 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002-wal-2.temp at 1732376775216 (+31 ms)3 split writer threads finished at 1732376775216Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732376775226 (+10 ms)Processed 1 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732376775259 (+33 ms) 2024-11-23T15:46:15,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:15,261 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:15,276 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal.1732376775263, exclude list is [], retry=0 2024-11-23T15:46:15,280 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:15,281 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:15,281 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:15,313 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal.1732376775263 2024-11-23T15:46:15,313 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:15,313 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b8b089c015a29e496a5185fa76e2f939, NAME => 'testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:15,314 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:15,314 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,314 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,316 INFO [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,318 INFO [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b089c015a29e496a5185fa76e2f939 columnFamilyName a 2024-11-23T15:46:15,318 DEBUG [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:15,319 INFO [StoreOpener-b8b089c015a29e496a5185fa76e2f939-1 {}] regionserver.HStore(327): Store=b8b089c015a29e496a5185fa76e2f939/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:15,320 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,326 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002 2024-11-23T15:46:15,354 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:15,360 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002 2024-11-23T15:46:15,365 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b8b089c015a29e496a5185fa76e2f939 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-23T15:46:15,431 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/.tmp/a/e5de20679a5e4a9a8941f3ad7492dbf9 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732376774931/Put/seqid=0 2024-11-23T15:46:15,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741844_1020 (size=5170) 2024-11-23T15:46:15,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741844_1020 (size=5170) 2024-11-23T15:46:15,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741844_1020 (size=5170) 2024-11-23T15:46:15,476 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/.tmp/a/e5de20679a5e4a9a8941f3ad7492dbf9 2024-11-23T15:46:15,523 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/.tmp/a/e5de20679a5e4a9a8941f3ad7492dbf9 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/a/e5de20679a5e4a9a8941f3ad7492dbf9 2024-11-23T15:46:15,532 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/a/e5de20679a5e4a9a8941f3ad7492dbf9, entries=2, sequenceid=2, filesize=5.0 K 2024-11-23T15:46:15,537 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b8b089c015a29e496a5185fa76e2f939 in 172ms, sequenceid=2, compaction requested=false; wal=null 2024-11-23T15:46:15,538 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/0000000000000000002 2024-11-23T15:46:15,539 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,539 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,542 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b8b089c015a29e496a5185fa76e2f939 2024-11-23T15:46:15,545 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/b8b089c015a29e496a5185fa76e2f939/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-23T15:46:15,546 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b8b089c015a29e496a5185fa76e2f939; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73976349, jitterRate=0.10233350098133087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:15,547 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b8b089c015a29e496a5185fa76e2f939: Writing region info on filesystem at 1732376775314Initializing all the Stores at 1732376775316 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376775316Obtaining lock to block concurrent updates at 1732376775365 (+49 ms)Preparing flush snapshotting stores in b8b089c015a29e496a5185fa76e2f939 at 1732376775365Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732376775369 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732376774828.b8b089c015a29e496a5185fa76e2f939. at 1732376775369Flushing b8b089c015a29e496a5185fa76e2f939/a: creating writer at 1732376775371 (+2 ms)Flushing b8b089c015a29e496a5185fa76e2f939/a: appending metadata at 1732376775416 (+45 ms)Flushing b8b089c015a29e496a5185fa76e2f939/a: closing flushed file at 1732376775419 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43cc758d: reopening flushed file at 1732376775521 (+102 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b8b089c015a29e496a5185fa76e2f939 in 172ms, sequenceid=2, compaction requested=false; wal=null at 1732376775537 (+16 ms)Cleaning up temporary data from old regions at 1732376775539 (+2 ms)Region opened successfully at 1732376775547 (+8 ms) 2024-11-23T15:46:15,569 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=379 (was 368) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:33634 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:44100 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33345/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:49084 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:33826 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:43940 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:49176 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=8079 (was 8195) 2024-11-23T15:46:15,580 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=379, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=8078 2024-11-23T15:46:15,598 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:15,600 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:15,601 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:15,605 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68430744, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-68430744, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:15,619 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68430744/hregion-68430744.1732376775605, exclude list is [], retry=0 2024-11-23T15:46:15,623 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:15,623 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:15,624 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:15,650 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68430744/hregion-68430744.1732376775605 2024-11-23T15:46:15,651 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:15,651 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 674ad99db159cca4d75354188c06a707, NAME => 'testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741846_1022 (size=64) 2024-11-23T15:46:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741846_1022 (size=64) 2024-11-23T15:46:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741846_1022 (size=64) 2024-11-23T15:46:15,664 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:15,666 INFO [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:15,669 INFO [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674ad99db159cca4d75354188c06a707 columnFamilyName a 2024-11-23T15:46:15,669 DEBUG [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:15,669 INFO [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] regionserver.HStore(327): Store=674ad99db159cca4d75354188c06a707/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:15,670 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:15,670 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707 2024-11-23T15:46:15,671 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707 2024-11-23T15:46:15,672 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:15,672 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:15,674 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:15,677 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:15,677 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 674ad99db159cca4d75354188c06a707; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71734849, jitterRate=0.06893254816532135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:15,678 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 674ad99db159cca4d75354188c06a707: Writing region info on filesystem at 1732376775664Initializing all the Stores at 1732376775665 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376775665Cleaning up temporary data from old regions at 1732376775672 (+7 ms)Region opened successfully at 1732376775678 (+6 ms) 2024-11-23T15:46:15,678 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 674ad99db159cca4d75354188c06a707, disabling compactions & flushes 2024-11-23T15:46:15,678 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707. 2024-11-23T15:46:15,678 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707. 2024-11-23T15:46:15,678 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707. after waiting 0 ms 2024-11-23T15:46:15,678 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707. 2024-11-23T15:46:15,678 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707. 2024-11-23T15:46:15,678 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 674ad99db159cca4d75354188c06a707: Waiting for close lock at 1732376775678Disabling compacts and flushes for region at 1732376775678Disabling writes for close at 1732376775678Writing region close event to WAL at 1732376775678Closed at 1732376775678 2024-11-23T15:46:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741845_1021 (size=175) 2024-11-23T15:46:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741845_1021 (size=175) 2024-11-23T15:46:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741845_1021 (size=175) 2024-11-23T15:46:15,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:15,684 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68430744:(num 1732376775605) 2024-11-23T15:46:15,685 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T15:46:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741847_1023 (size=469) 2024-11-23T15:46:15,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741847_1023 (size=469) 2024-11-23T15:46:15,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741847_1023 (size=469) 2024-11-23T15:46:15,728 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-23T15:46:15,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741848_1024 (size=322) 2024-11-23T15:46:15,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741848_1024 (size=322) 2024-11-23T15:46:15,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741848_1024 (size=322) 2024-11-23T15:46:15,779 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2, size=322 (322bytes) 2024-11-23T15:46:15,779 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2 2024-11-23T15:46:15,780 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2 after 1ms 2024-11-23T15:46:15,808 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:15,808 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2 took 29ms 2024-11-23T15:46:15,811 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2 so closing down 2024-11-23T15:46:15,811 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:15,813 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-23T15:46:15,838 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002-wal-2.temp 2024-11-23T15:46:15,838 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741849_1025 (size=322) 2024-11-23T15:46:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741849_1025 (size=322) 2024-11-23T15:46:15,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741849_1025 (size=322) 2024-11-23T15:46:15,847 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:15,850 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002 2024-11-23T15:46:15,850 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-23T15:46:15,850 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2, journal: Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2, size=322 (322bytes) at 1732376775779Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2 so closing down at 1732376775811 (+32 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002-wal-2.temp at 1732376775838 (+27 ms)3 split writer threads finished at 1732376775838Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732376775847 (+9 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002 at 1732376775850 (+3 ms)Processed 1 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732376775850 2024-11-23T15:46:15,864 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1, size=469 (469bytes) 2024-11-23T15:46:15,864 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1 2024-11-23T15:46:15,864 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1 after 0ms 2024-11-23T15:46:15,889 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:15,889 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1 took 26ms 2024-11-23T15:46:15,892 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1 so closing down 2024-11-23T15:46:15,892 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:15,895 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-23T15:46:15,923 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000001-wal-1.temp 2024-11-23T15:46:15,923 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:15,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741850_1026 (size=469) 2024-11-23T15:46:15,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741850_1026 (size=469) 2024-11-23T15:46:15,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741850_1026 (size=469) 2024-11-23T15:46:15,932 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:15,959 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:15,962 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002, length=322 2024-11-23T15:46:15,964 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002 2024-11-23T15:46:15,964 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-23T15:46:15,965 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1, journal: Splitting hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1, size=469 (469bytes) at 1732376775864Finishing writing output for hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1 so closing down at 1732376775892 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000001-wal-1.temp at 1732376775923 (+31 ms)3 split writer threads finished at 1732376775923Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732376775932 (+9 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002 at 1732376775964 (+32 ms)Processed 2 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732376775964 2024-11-23T15:46:15,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:15,967 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:15,978 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal.1732376775967, exclude list is [], retry=0 2024-11-23T15:46:15,982 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:15,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:15,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:16,007 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal.1732376775967 2024-11-23T15:46:16,007 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:16,007 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 674ad99db159cca4d75354188c06a707, NAME => 'testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:16,008 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:16,008 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,008 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,010 INFO [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,011 INFO [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674ad99db159cca4d75354188c06a707 columnFamilyName a 2024-11-23T15:46:16,012 DEBUG [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:16,012 INFO [StoreOpener-674ad99db159cca4d75354188c06a707-1 {}] regionserver.HStore(327): Store=674ad99db159cca4d75354188c06a707/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:16,012 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,013 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,016 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,017 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002 2024-11-23T15:46:16,046 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:16,047 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002 2024-11-23T15:46:16,048 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 674ad99db159cca4d75354188c06a707 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-23T15:46:16,063 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/.tmp/a/fa22ac0780764075851bf8052a9a639e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732376775684/Put/seqid=0 2024-11-23T15:46:16,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741852_1028 (size=5170) 2024-11-23T15:46:16,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741852_1028 (size=5170) 2024-11-23T15:46:16,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741852_1028 (size=5170) 2024-11-23T15:46:16,071 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/.tmp/a/fa22ac0780764075851bf8052a9a639e 2024-11-23T15:46:16,079 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/.tmp/a/fa22ac0780764075851bf8052a9a639e as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/a/fa22ac0780764075851bf8052a9a639e 2024-11-23T15:46:16,087 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/a/fa22ac0780764075851bf8052a9a639e, entries=2, sequenceid=2, filesize=5.0 K 2024-11-23T15:46:16,087 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 674ad99db159cca4d75354188c06a707 in 39ms, sequenceid=2, compaction requested=false; wal=null 2024-11-23T15:46:16,088 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/0000000000000000002 2024-11-23T15:46:16,089 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,089 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,091 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 674ad99db159cca4d75354188c06a707 2024-11-23T15:46:16,094 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/674ad99db159cca4d75354188c06a707/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-23T15:46:16,095 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 674ad99db159cca4d75354188c06a707; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64158444, jitterRate=-0.043964684009552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:16,095 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 674ad99db159cca4d75354188c06a707: Writing region info on filesystem at 1732376776008Initializing all the Stores at 1732376776009 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376776009Obtaining lock to block concurrent updates at 1732376776048 (+39 ms)Preparing flush snapshotting stores in 674ad99db159cca4d75354188c06a707 at 1732376776048Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732376776048Flushing stores of testReplayEditsWrittenIntoWAL,,1732376775598.674ad99db159cca4d75354188c06a707. at 1732376776048Flushing 674ad99db159cca4d75354188c06a707/a: creating writer at 1732376776048Flushing 674ad99db159cca4d75354188c06a707/a: appending metadata at 1732376776062 (+14 ms)Flushing 674ad99db159cca4d75354188c06a707/a: closing flushed file at 1732376776062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79acb157: reopening flushed file at 1732376776078 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 674ad99db159cca4d75354188c06a707 in 39ms, sequenceid=2, compaction requested=false; wal=null at 1732376776087 (+9 ms)Cleaning up temporary data from old regions at 1732376776089 (+2 ms)Region opened successfully at 1732376776095 (+6 ms) 2024-11-23T15:46:16,118 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=389 (was 379) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:33634 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:44158 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:49242 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:33896 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:49084 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:43940 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=8065 (was 8078) 2024-11-23T15:46:16,131 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=389, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=8062 2024-11-23T15:46:16,147 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:16,149 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:16,150 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:16,154 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50074479, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-50074479, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:16,166 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50074479/hregion-50074479.1732376776154, exclude list is [], retry=0 2024-11-23T15:46:16,170 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:16,170 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:16,171 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:16,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50074479/hregion-50074479.1732376776154 2024-11-23T15:46:16,197 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:16,197 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 87d63f4c6981f931151781488e727118, NAME => 'testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:16,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741854_1030 (size=64) 2024-11-23T15:46:16,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741854_1030 (size=64) 2024-11-23T15:46:16,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741854_1030 (size=64) 2024-11-23T15:46:16,209 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:16,211 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,213 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87d63f4c6981f931151781488e727118 columnFamilyName a 2024-11-23T15:46:16,213 DEBUG [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:16,214 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(327): Store=87d63f4c6981f931151781488e727118/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:16,214 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,216 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87d63f4c6981f931151781488e727118 columnFamilyName b 2024-11-23T15:46:16,216 DEBUG [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:16,217 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(327): Store=87d63f4c6981f931151781488e727118/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:16,217 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,219 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87d63f4c6981f931151781488e727118 columnFamilyName c 2024-11-23T15:46:16,219 DEBUG [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:16,219 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(327): Store=87d63f4c6981f931151781488e727118/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:16,220 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,221 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,221 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,223 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,223 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,223 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:16,225 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:16,229 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:16,229 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 87d63f4c6981f931151781488e727118; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64956638, jitterRate=-0.032070666551589966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:16,230 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 87d63f4c6981f931151781488e727118: Writing region info on filesystem at 1732376776209Initializing all the Stores at 1732376776210 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376776210Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376776211 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376776211Cleaning up temporary data from old regions at 1732376776223 (+12 ms)Region opened successfully at 1732376776230 (+7 ms) 2024-11-23T15:46:16,230 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 87d63f4c6981f931151781488e727118, disabling compactions & flushes 2024-11-23T15:46:16,230 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:16,230 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:16,230 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. after waiting 0 ms 2024-11-23T15:46:16,230 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:16,231 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:16,231 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 87d63f4c6981f931151781488e727118: Waiting for close lock at 1732376776230Disabling compacts and flushes for region at 1732376776230Disabling writes for close at 1732376776230Writing region close event to WAL at 1732376776231 (+1 ms)Closed at 1732376776231 2024-11-23T15:46:16,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741853_1029 (size=175) 2024-11-23T15:46:16,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741853_1029 (size=175) 2024-11-23T15:46:16,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741853_1029 (size=175) 2024-11-23T15:46:16,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:16,238 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50074479:(num 1732376776154) 2024-11-23T15:46:16,238 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:16,241 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:16,252 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241, exclude list is [], retry=0 2024-11-23T15:46:16,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:16,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:16,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:16,282 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 2024-11-23T15:46:16,282 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309)] 2024-11-23T15:46:16,509 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241, size=0 (0bytes) 2024-11-23T15:46:16,509 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 might be still open, length is 0 2024-11-23T15:46:16,509 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 2024-11-23T15:46:16,510 WARN [IPC Server handler 1 on default port 33345 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-23T15:46:16,511 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 after 2ms 2024-11-23T15:46:16,596 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:44196 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44917:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44196 dst: /127.0.0.1:44917 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44917 remote=/127.0.0.1:44196]. Total timeout mills is 60000, 59885 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:16,597 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:49274 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49274 dst: /127.0.0.1:35471 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:16,597 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:33942 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37735:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33942 dst: /127.0.0.1:37735 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:16,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741855_1032 (size=497775) 2024-11-23T15:46:16,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741855_1032 (size=497775) 2024-11-23T15:46:16,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741855_1032 (size=497775) 2024-11-23T15:46:20,007 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:46:20,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T15:46:20,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 after 4003ms 2024-11-23T15:46:20,547 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:20,548 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 took 4039ms 2024-11-23T15:46:20,553 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732376776241.temp 2024-11-23T15:46:20,579 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000000001-wal.1732376776241.temp 2024-11-23T15:46:20,646 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241; continuing. 2024-11-23T15:46:20,646 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 so closing down 2024-11-23T15:46:20,646 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:20,647 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:20,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741856_1033 (size=497783) 2024-11-23T15:46:20,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741856_1033 (size=497783) 2024-11-23T15:46:20,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741856_1033 (size=497783) 2024-11-23T15:46:20,651 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000000001-wal.1732376776241.temp (wrote 3002 edits, skipped 0 edits in 45 ms) 2024-11-23T15:46:20,654 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000000001-wal.1732376776241.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000003002 2024-11-23T15:46:20,654 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 105 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T15:46:20,654 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241, journal: Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241, size=0 (0bytes) at 1732376776509Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000000001-wal.1732376776241.temp at 1732376780579 (+4070 ms)Split 1024 edits, skipped 0 edits. at 1732376780603 (+24 ms)Split 2048 edits, skipped 0 edits. at 1732376780626 (+23 ms)Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 so closing down at 1732376780646 (+20 ms)3 split writer threads finished at 1732376780647 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000000001-wal.1732376776241.temp (wrote 3002 edits, skipped 0 edits in 45 ms) at 1732376780652 (+5 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000000001-wal.1732376776241.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000003002 at 1732376780654 (+2 ms)Processed 3002 edits across 1 Regions in 105 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241, size=0, length=0, corrupted=false, cancelled=false at 1732376780654 2024-11-23T15:46:20,656 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376776241 2024-11-23T15:46:20,658 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000003002 2024-11-23T15:46:20,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:20,661 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:20,674 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376780662, exclude list is [], retry=0 2024-11-23T15:46:20,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:20,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:20,679 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:20,705 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376780662 2024-11-23T15:46:20,706 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:20,706 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:20,708 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 87d63f4c6981f931151781488e727118 2024-11-23T15:46:20,709 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87d63f4c6981f931151781488e727118 columnFamilyName a 2024-11-23T15:46:20,709 DEBUG [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:20,710 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(327): Store=87d63f4c6981f931151781488e727118/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:20,710 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 87d63f4c6981f931151781488e727118 2024-11-23T15:46:20,711 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87d63f4c6981f931151781488e727118 columnFamilyName b 2024-11-23T15:46:20,712 DEBUG [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:20,712 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(327): Store=87d63f4c6981f931151781488e727118/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:20,712 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 87d63f4c6981f931151781488e727118 2024-11-23T15:46:20,714 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87d63f4c6981f931151781488e727118 columnFamilyName c 2024-11-23T15:46:20,714 DEBUG [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:20,714 INFO [StoreOpener-87d63f4c6981f931151781488e727118-1 {}] regionserver.HStore(327): Store=87d63f4c6981f931151781488e727118/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:20,715 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:20,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118 2024-11-23T15:46:20,718 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118 2024-11-23T15:46:20,719 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000003002 2024-11-23T15:46:20,746 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:20,785 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T15:46:21,116 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 87d63f4c6981f931151781488e727118 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T15:46:21,165 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/a/43440354621546d795231a677181db7c is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732376776290/Put/seqid=0 2024-11-23T15:46:21,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741858_1035 (size=50463) 2024-11-23T15:46:21,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741858_1035 (size=50463) 2024-11-23T15:46:21,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741858_1035 (size=50463) 2024-11-23T15:46:21,177 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/a/43440354621546d795231a677181db7c 2024-11-23T15:46:21,186 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/a/43440354621546d795231a677181db7c as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/a/43440354621546d795231a677181db7c 2024-11-23T15:46:21,194 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/a/43440354621546d795231a677181db7c, entries=754, sequenceid=754, filesize=49.3 K 2024-11-23T15:46:21,194 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 87d63f4c6981f931151781488e727118 in 78ms, sequenceid=754, compaction requested=false; wal=null 2024-11-23T15:46:21,207 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T15:46:21,207 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 87d63f4c6981f931151781488e727118 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T15:46:21,217 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/a/459c7c89d0a14bd6a4eedd5c72c9d276 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732376776337/Put/seqid=0 2024-11-23T15:46:21,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741859_1036 (size=20072) 2024-11-23T15:46:21,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741859_1036 (size=20072) 2024-11-23T15:46:21,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741859_1036 (size=20072) 2024-11-23T15:46:21,227 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/a/459c7c89d0a14bd6a4eedd5c72c9d276 2024-11-23T15:46:21,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/b/778d8359938941eeaa8468105b0ed6c5 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732376776366/Put/seqid=0 2024-11-23T15:46:21,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741860_1037 (size=35835) 2024-11-23T15:46:21,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741860_1037 (size=35835) 2024-11-23T15:46:21,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741860_1037 (size=35835) 2024-11-23T15:46:21,261 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/b/778d8359938941eeaa8468105b0ed6c5 2024-11-23T15:46:21,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/a/459c7c89d0a14bd6a4eedd5c72c9d276 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/a/459c7c89d0a14bd6a4eedd5c72c9d276 2024-11-23T15:46:21,278 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/a/459c7c89d0a14bd6a4eedd5c72c9d276, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-23T15:46:21,280 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/b/778d8359938941eeaa8468105b0ed6c5 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/b/778d8359938941eeaa8468105b0ed6c5 2024-11-23T15:46:21,286 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/b/778d8359938941eeaa8468105b0ed6c5, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-23T15:46:21,287 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 87d63f4c6981f931151781488e727118 in 80ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-23T15:46:21,298 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T15:46:21,299 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 87d63f4c6981f931151781488e727118 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-23T15:46:21,306 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/b/1bcbc56476ad4a5c9e3824952a546acf is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732376776395/Put/seqid=0 2024-11-23T15:46:21,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741861_1038 (size=35082) 2024-11-23T15:46:21,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741861_1038 (size=35082) 2024-11-23T15:46:21,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741861_1038 (size=35082) 2024-11-23T15:46:21,319 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/b/1bcbc56476ad4a5c9e3824952a546acf 2024-11-23T15:46:21,342 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/c/a1cddaf8c565426ba15c8fd67503906b is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732376776427/Put/seqid=0 2024-11-23T15:46:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741862_1039 (size=20825) 2024-11-23T15:46:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741862_1039 (size=20825) 2024-11-23T15:46:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741862_1039 (size=20825) 2024-11-23T15:46:21,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/c/a1cddaf8c565426ba15c8fd67503906b 2024-11-23T15:46:21,360 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/b/1bcbc56476ad4a5c9e3824952a546acf as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/b/1bcbc56476ad4a5c9e3824952a546acf 2024-11-23T15:46:21,367 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/b/1bcbc56476ad4a5c9e3824952a546acf, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-23T15:46:21,369 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/c/a1cddaf8c565426ba15c8fd67503906b as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/c/a1cddaf8c565426ba15c8fd67503906b 2024-11-23T15:46:21,376 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/c/a1cddaf8c565426ba15c8fd67503906b, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-23T15:46:21,377 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 87d63f4c6981f931151781488e727118 in 78ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-23T15:46:21,390 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732376776480/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:21,393 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000003002 2024-11-23T15:46:21,394 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-23T15:46:21,394 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 87d63f4c6981f931151781488e727118 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-23T15:46:21,406 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/c/f896da16a04f4e28992160a65ba66896 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732376776435/Put/seqid=0 2024-11-23T15:46:21,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741863_1040 (size=50301) 2024-11-23T15:46:21,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741863_1040 (size=50301) 2024-11-23T15:46:21,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741863_1040 (size=50301) 2024-11-23T15:46:21,421 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/c/f896da16a04f4e28992160a65ba66896 2024-11-23T15:46:21,428 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f896da16a04f4e28992160a65ba66896 2024-11-23T15:46:21,430 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/.tmp/c/f896da16a04f4e28992160a65ba66896 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/c/f896da16a04f4e28992160a65ba66896 2024-11-23T15:46:21,437 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f896da16a04f4e28992160a65ba66896 2024-11-23T15:46:21,437 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/c/f896da16a04f4e28992160a65ba66896, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-23T15:46:21,438 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 87d63f4c6981f931151781488e727118 in 44ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-23T15:46:21,439 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/0000000000000003002 2024-11-23T15:46:21,440 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:21,440 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:21,441 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:46:21,443 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 87d63f4c6981f931151781488e727118 2024-11-23T15:46:21,446 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenIntoWAL/87d63f4c6981f931151781488e727118/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-23T15:46:21,447 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 87d63f4c6981f931151781488e727118; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59175667, jitterRate=-0.11821384727954865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:46:21,447 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 87d63f4c6981f931151781488e727118: Writing region info on filesystem at 1732376780706Initializing all the Stores at 1732376780707 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376780708 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376780708Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376780708Cleaning up temporary data from old regions at 1732376781440 (+732 ms)Region opened successfully at 1732376781447 (+7 ms) 2024-11-23T15:46:21,519 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 87d63f4c6981f931151781488e727118, disabling compactions & flushes 2024-11-23T15:46:21,519 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:21,519 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:21,519 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. after waiting 0 ms 2024-11-23T15:46:21,519 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:21,522 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732376776148.87d63f4c6981f931151781488e727118. 2024-11-23T15:46:21,522 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 87d63f4c6981f931151781488e727118: Waiting for close lock at 1732376781519Disabling compacts and flushes for region at 1732376781519Disabling writes for close at 1732376781519Writing region close event to WAL at 1732376781521 (+2 ms)Closed at 1732376781521 2024-11-23T15:46:21,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741857_1034 (size=175) 2024-11-23T15:46:21,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741857_1034 (size=175) 2024-11-23T15:46:21,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741857_1034 (size=175) 2024-11-23T15:46:21,529 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:21,529 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732376780662) 2024-11-23T15:46:21,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=403 (was 389) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:46721 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46721 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33345 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:45139 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45139 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408699297_22 at /127.0.0.1:33976 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408699297_22 at /127.0.0.1:49300 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408699297_22 at /127.0.0.1:44242 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408699297_22 at /127.0.0.1:34000 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:33345 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 247) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8000 (was 8062) 2024-11-23T15:46:21,560 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=7998 2024-11-23T15:46:21,576 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:21,579 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:21,580 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:21,583 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-00729145, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-00729145, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:21,597 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-00729145/hregion-00729145.1732376781584, exclude list is [], retry=0 2024-11-23T15:46:21,600 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:21,600 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:21,601 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:21,629 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-00729145/hregion-00729145.1732376781584 2024-11-23T15:46:21,629 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:21,629 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a4d49201499a47eb86586186eb4b92a7, NAME => 'test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:21,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741865_1042 (size=43) 2024-11-23T15:46:21,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741865_1042 (size=43) 2024-11-23T15:46:21,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741865_1042 (size=43) 2024-11-23T15:46:21,641 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:21,642 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,644 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d49201499a47eb86586186eb4b92a7 columnFamilyName a 2024-11-23T15:46:21,644 DEBUG [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:21,645 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(327): Store=a4d49201499a47eb86586186eb4b92a7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:21,645 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,647 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d49201499a47eb86586186eb4b92a7 columnFamilyName b 2024-11-23T15:46:21,647 DEBUG [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:21,647 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(327): Store=a4d49201499a47eb86586186eb4b92a7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:21,647 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,649 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d49201499a47eb86586186eb4b92a7 columnFamilyName c 2024-11-23T15:46:21,649 DEBUG [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:21,650 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(327): Store=a4d49201499a47eb86586186eb4b92a7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:21,650 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,653 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,653 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,654 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:21,655 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:21,659 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:21,660 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d49201499a47eb86586186eb4b92a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65555362, jitterRate=-0.02314898371696472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:21,661 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d49201499a47eb86586186eb4b92a7: Writing region info on filesystem at 1732376781641Initializing all the Stores at 1732376781642 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376781642Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376781642Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376781642Cleaning up temporary data from old regions at 1732376781653 (+11 ms)Region opened successfully at 1732376781661 (+8 ms) 2024-11-23T15:46:21,661 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4d49201499a47eb86586186eb4b92a7, disabling compactions & flushes 2024-11-23T15:46:21,661 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:21,662 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:21,662 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. after waiting 0 ms 2024-11-23T15:46:21,662 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:21,662 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:21,662 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4d49201499a47eb86586186eb4b92a7: Waiting for close lock at 1732376781661Disabling compacts and flushes for region at 1732376781661Disabling writes for close at 1732376781662 (+1 ms)Writing region close event to WAL at 1732376781662Closed at 1732376781662 2024-11-23T15:46:21,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741864_1041 (size=175) 2024-11-23T15:46:21,667 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-00729145/hregion-00729145.1732376781584 not finished, retry = 0 2024-11-23T15:46:21,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741864_1041 (size=175) 2024-11-23T15:46:21,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741864_1041 (size=175) 2024-11-23T15:46:21,770 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:21,770 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-00729145:(num 1732376781584) 2024-11-23T15:46:21,771 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:21,773 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:21,791 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774, exclude list is [], retry=0 2024-11-23T15:46:21,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:21,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:21,796 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:21,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T15:46:21,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:21,819 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:46:21,819 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T15:46:21,819 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T15:46:21,819 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:21,820 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-23T15:46:21,820 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:21,820 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774 2024-11-23T15:46:21,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309)] 2024-11-23T15:46:21,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741866_1043 (size=371388) 2024-11-23T15:46:21,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741866_1043 (size=371388) 2024-11-23T15:46:21,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741866_1043 (size=371388) 2024-11-23T15:46:22,003 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774, size=362.7 K (371388bytes) 2024-11-23T15:46:22,003 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774 2024-11-23T15:46:22,004 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774 after 1ms 2024-11-23T15:46:22,033 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:22,035 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774 took 32ms 2024-11-23T15:46:22,039 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732376781774.temp 2024-11-23T15:46:22,074 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774 so closing down 2024-11-23T15:46:22,075 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:22,079 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000000001-wal.1732376781774.temp 2024-11-23T15:46:22,095 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:22,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741867_1044 (size=371388) 2024-11-23T15:46:22,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741867_1044 (size=371388) 2024-11-23T15:46:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741867_1044 (size=371388) 2024-11-23T15:46:22,099 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000000001-wal.1732376781774.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-11-23T15:46:22,101 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000000001-wal.1732376781774.temp to hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003000 2024-11-23T15:46:22,101 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-23T15:46:22,102 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774, journal: Splitting hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774, size=362.7 K (371388bytes) at 1732376782003Split 1024 edits, skipped 0 edits. at 1732376782051 (+48 ms)Split 2048 edits, skipped 0 edits. at 1732376782064 (+13 ms)Finishing writing output for hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774 so closing down at 1732376782074 (+10 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000000001-wal.1732376781774.temp at 1732376782079 (+5 ms)3 split writer threads finished at 1732376782095 (+16 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000000001-wal.1732376781774.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1732376782099 (+4 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000000001-wal.1732376781774.temp to hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003000 at 1732376782101 (+2 ms)Processed 3000 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732376782101 2024-11-23T15:46:22,104 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376781774 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376781774 2024-11-23T15:46:22,105 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003000 2024-11-23T15:46:22,105 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:22,108 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:22,127 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108, exclude list is [], retry=0 2024-11-23T15:46:22,130 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:22,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:22,132 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:22,161 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108 2024-11-23T15:46:22,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:22,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741868_1045 (size=371515) 2024-11-23T15:46:22,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741868_1045 (size=371515) 2024-11-23T15:46:22,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741868_1045 (size=371515) 2024-11-23T15:46:22,389 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108, size=362.8 K (371515bytes) 2024-11-23T15:46:22,389 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108 2024-11-23T15:46:22,390 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108 after 1ms 2024-11-23T15:46:22,432 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:22,434 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108 took 45ms 2024-11-23T15:46:22,438 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732376782108.temp 2024-11-23T15:46:22,471 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003001-wal.1732376782108.temp 2024-11-23T15:46:22,477 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108 so closing down 2024-11-23T15:46:22,477 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:22,485 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:22,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741869_1046 (size=371515) 2024-11-23T15:46:22,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741869_1046 (size=371515) 2024-11-23T15:46:22,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741869_1046 (size=371515) 2024-11-23T15:46:22,490 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003001-wal.1732376782108.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-11-23T15:46:22,491 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003001-wal.1732376782108.temp to hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000006000 2024-11-23T15:46:22,492 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 58 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-23T15:46:22,492 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108, journal: Splitting hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108, size=362.8 K (371515bytes) at 1732376782389Split 1024 edits, skipped 0 edits. at 1732376782449 (+60 ms)Split 2048 edits, skipped 0 edits. at 1732376782463 (+14 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003001-wal.1732376782108.temp at 1732376782471 (+8 ms)Finishing writing output for hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108 so closing down at 1732376782477 (+6 ms)3 split writer threads finished at 1732376782485 (+8 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003001-wal.1732376782108.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1732376782490 (+5 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003001-wal.1732376782108.temp to hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000006000 at 1732376782492 (+2 ms)Processed 3000 edits across 1 Regions in 58 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732376782492 2024-11-23T15:46:22,494 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782108 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376782108 2024-11-23T15:46:22,495 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000006000 2024-11-23T15:46:22,496 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:22,498 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/test2727-manual,16010,1732376781575, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:22,517 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782499, exclude list is [], retry=0 2024-11-23T15:46:22,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:22,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:22,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:22,564 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732376781575/wal.1732376782499 2024-11-23T15:46:22,565 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:22,565 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4d49201499a47eb86586186eb4b92a7, NAME => 'test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:22,566 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:22,566 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,566 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,568 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,569 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d49201499a47eb86586186eb4b92a7 columnFamilyName a 2024-11-23T15:46:22,569 DEBUG [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:22,570 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(327): Store=a4d49201499a47eb86586186eb4b92a7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:22,570 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,572 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d49201499a47eb86586186eb4b92a7 columnFamilyName b 2024-11-23T15:46:22,572 DEBUG [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:22,572 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(327): Store=a4d49201499a47eb86586186eb4b92a7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:22,572 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,574 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d49201499a47eb86586186eb4b92a7 columnFamilyName c 2024-11-23T15:46:22,574 DEBUG [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:22,575 INFO [StoreOpener-a4d49201499a47eb86586186eb4b92a7-1 {}] regionserver.HStore(327): Store=a4d49201499a47eb86586186eb4b92a7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:22,575 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,576 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:22,581 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003000 2024-11-23T15:46:22,611 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:22,644 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003000 2024-11-23T15:46:22,646 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000006000 2024-11-23T15:46:22,676 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:22,714 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000006000 2024-11-23T15:46:22,715 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4d49201499a47eb86586186eb4b92a7 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-23T15:46:22,758 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/a/d8451d68c6464b03a01df8b50d14d82e is 41, key is test2727/a:100/1732376782166/Put/seqid=0 2024-11-23T15:46:22,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741871_1048 (size=84227) 2024-11-23T15:46:22,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741871_1048 (size=84227) 2024-11-23T15:46:22,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741871_1048 (size=84227) 2024-11-23T15:46:23,168 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/a/d8451d68c6464b03a01df8b50d14d82e 2024-11-23T15:46:23,205 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/b/3ba661ac8f0041baadc02e0246da7f94 is 41, key is test2727/b:100/1732376782236/Put/seqid=0 2024-11-23T15:46:23,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741872_1049 (size=84609) 2024-11-23T15:46:23,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741872_1049 (size=84609) 2024-11-23T15:46:23,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741872_1049 (size=84609) 2024-11-23T15:46:23,614 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/b/3ba661ac8f0041baadc02e0246da7f94 2024-11-23T15:46:23,641 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/c/25af2a9c398441b0b56b40bec4ea6d0c is 41, key is test2727/c:100/1732376782309/Put/seqid=0 2024-11-23T15:46:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741873_1050 (size=84609) 2024-11-23T15:46:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741873_1050 (size=84609) 2024-11-23T15:46:23,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741873_1050 (size=84609) 2024-11-23T15:46:23,648 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/c/25af2a9c398441b0b56b40bec4ea6d0c 2024-11-23T15:46:23,655 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/a/d8451d68c6464b03a01df8b50d14d82e as hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/a/d8451d68c6464b03a01df8b50d14d82e 2024-11-23T15:46:23,662 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/a/d8451d68c6464b03a01df8b50d14d82e, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-23T15:46:23,664 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/b/3ba661ac8f0041baadc02e0246da7f94 as hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/b/3ba661ac8f0041baadc02e0246da7f94 2024-11-23T15:46:23,670 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/b/3ba661ac8f0041baadc02e0246da7f94, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-23T15:46:23,672 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/.tmp/c/25af2a9c398441b0b56b40bec4ea6d0c as hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/c/25af2a9c398441b0b56b40bec4ea6d0c 2024-11-23T15:46:23,678 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/c/25af2a9c398441b0b56b40bec4ea6d0c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-23T15:46:23,678 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a4d49201499a47eb86586186eb4b92a7 in 964ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-23T15:46:23,679 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000003000 2024-11-23T15:46:23,679 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/0000000000000006000 2024-11-23T15:46:23,680 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:23,680 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:23,681 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:23,682 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d49201499a47eb86586186eb4b92a7 2024-11-23T15:46:23,684 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/test2727/a4d49201499a47eb86586186eb4b92a7/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-23T15:46:23,685 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d49201499a47eb86586186eb4b92a7; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74122684, jitterRate=0.10451406240463257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:23,686 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d49201499a47eb86586186eb4b92a7: Writing region info on filesystem at 1732376782566Initializing all the Stores at 1732376782567 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376782567Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376782568 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376782568Obtaining lock to block concurrent updates at 1732376782715 (+147 ms)Preparing flush snapshotting stores in a4d49201499a47eb86586186eb4b92a7 at 1732376782715Finished memstore snapshotting test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732376782715Flushing stores of test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. at 1732376782715Flushing a4d49201499a47eb86586186eb4b92a7/a: creating writer at 1732376782715Flushing a4d49201499a47eb86586186eb4b92a7/a: appending metadata at 1732376782757 (+42 ms)Flushing a4d49201499a47eb86586186eb4b92a7/a: closing flushed file at 1732376782757Flushing a4d49201499a47eb86586186eb4b92a7/b: creating writer at 1732376783182 (+425 ms)Flushing a4d49201499a47eb86586186eb4b92a7/b: appending metadata at 1732376783203 (+21 ms)Flushing a4d49201499a47eb86586186eb4b92a7/b: closing flushed file at 1732376783203Flushing a4d49201499a47eb86586186eb4b92a7/c: creating writer at 1732376783624 (+421 ms)Flushing a4d49201499a47eb86586186eb4b92a7/c: appending metadata at 1732376783640 (+16 ms)Flushing a4d49201499a47eb86586186eb4b92a7/c: closing flushed file at 1732376783640Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@584155d1: reopening flushed file at 1732376783654 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1a90d2: reopening flushed file at 1732376783663 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70cecd15: reopening flushed file at 1732376783670 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a4d49201499a47eb86586186eb4b92a7 in 964ms, sequenceid=6000, compaction requested=false; wal=null at 1732376783678 (+8 ms)Cleaning up temporary data from old regions at 1732376783680 (+2 ms)Region opened successfully at 1732376783686 (+6 ms) 2024-11-23T15:46:23,687 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-23T15:46:23,687 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4d49201499a47eb86586186eb4b92a7, disabling compactions & flushes 2024-11-23T15:46:23,687 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:23,687 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:23,687 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. after waiting 0 ms 2024-11-23T15:46:23,687 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:23,688 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732376781577.a4d49201499a47eb86586186eb4b92a7. 2024-11-23T15:46:23,689 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4d49201499a47eb86586186eb4b92a7: Waiting for close lock at 1732376783687Disabling compacts and flushes for region at 1732376783687Disabling writes for close at 1732376783687Writing region close event to WAL at 1732376783688 (+1 ms)Closed at 1732376783688 2024-11-23T15:46:23,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741870_1047 (size=175) 2024-11-23T15:46:23,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741870_1047 (size=175) 2024-11-23T15:46:23,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741870_1047 (size=175) 2024-11-23T15:46:23,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:23,696 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732376782499) 2024-11-23T15:46:23,706 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=408 (was 403) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:33976 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:34000 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:55312 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:38774 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 301), ProcessCount=11 (was 11), AvailableMemoryMB=7764 (was 7998) 2024-11-23T15:46:23,717 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=7763 2024-11-23T15:46:23,733 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:23,739 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:23,740 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732376783740 2024-11-23T15:46:23,770 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740 2024-11-23T15:46:23,774 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:23,778 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d549dc371d24b9be39236f3680b6fe8e, NAME => 'testSequentialEditLogSeqNum,,1732376783734.d549dc371d24b9be39236f3680b6fe8e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:23,778 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732376783734.d549dc371d24b9be39236f3680b6fe8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:23,778 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,778 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,779 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e doesn't exist for region: d549dc371d24b9be39236f3680b6fe8e on table testSequentialEditLogSeqNum 2024-11-23T15:46:23,780 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: d549dc371d24b9be39236f3680b6fe8e on table testSequentialEditLogSeqNum 2024-11-23T15:46:23,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741875_1052 (size=62) 2024-11-23T15:46:23,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741875_1052 (size=62) 2024-11-23T15:46:23,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741875_1052 (size=62) 2024-11-23T15:46:23,794 INFO [StoreOpener-d549dc371d24b9be39236f3680b6fe8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,796 INFO [StoreOpener-d549dc371d24b9be39236f3680b6fe8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d549dc371d24b9be39236f3680b6fe8e columnFamilyName a 2024-11-23T15:46:23,796 DEBUG [StoreOpener-d549dc371d24b9be39236f3680b6fe8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:23,796 INFO [StoreOpener-d549dc371d24b9be39236f3680b6fe8e-1 {}] regionserver.HStore(327): Store=d549dc371d24b9be39236f3680b6fe8e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:23,796 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,797 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,798 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,798 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,798 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,800 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d549dc371d24b9be39236f3680b6fe8e 2024-11-23T15:46:23,803 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:23,803 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d549dc371d24b9be39236f3680b6fe8e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68461369, jitterRate=0.020153895020484924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:23,805 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d549dc371d24b9be39236f3680b6fe8e: Writing region info on filesystem at 1732376783779Initializing all the Stores at 1732376783793 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376783793Cleaning up temporary data from old regions at 1732376783798 (+5 ms)Region opened successfully at 1732376783805 (+7 ms) 2024-11-23T15:46:23,819 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d549dc371d24b9be39236f3680b6fe8e 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-23T15:46:23,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/.tmp/a/e934797f6fe84b409de85ae69a628604 is 81, key is testSequentialEditLogSeqNum/a:x0/1732376783805/Put/seqid=0 2024-11-23T15:46:23,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741876_1053 (size=5833) 2024-11-23T15:46:23,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741876_1053 (size=5833) 2024-11-23T15:46:23,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741876_1053 (size=5833) 2024-11-23T15:46:23,853 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/.tmp/a/e934797f6fe84b409de85ae69a628604 2024-11-23T15:46:23,859 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/.tmp/a/e934797f6fe84b409de85ae69a628604 as hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/a/e934797f6fe84b409de85ae69a628604 2024-11-23T15:46:23,865 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/a/e934797f6fe84b409de85ae69a628604, entries=10, sequenceid=13, filesize=5.7 K 2024-11-23T15:46:23,867 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d549dc371d24b9be39236f3680b6fe8e in 48ms, sequenceid=13, compaction requested=false 2024-11-23T15:46:23,867 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d549dc371d24b9be39236f3680b6fe8e: 2024-11-23T15:46:23,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T15:46:23,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T15:46:23,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T15:46:23,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T15:46:23,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-23T15:46:23,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741874_1051 (size=3256) 2024-11-23T15:46:23,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741874_1051 (size=3256) 2024-11-23T15:46:23,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741874_1051 (size=3256) 2024-11-23T15:46:23,889 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740, size=3.2 K (3256bytes) 2024-11-23T15:46:23,889 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740 2024-11-23T15:46:23,890 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740 after 1ms 2024-11-23T15:46:23,916 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:23,917 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740 took 28ms 2024-11-23T15:46:23,920 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740 so closing down 2024-11-23T15:46:23,920 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:23,921 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732376783740.temp 2024-11-23T15:46:23,951 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000003-wal.1732376783740.temp 2024-11-23T15:46:23,952 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:23,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741877_1054 (size=2650) 2024-11-23T15:46:23,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741877_1054 (size=2650) 2024-11-23T15:46:23,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741877_1054 (size=2650) 2024-11-23T15:46:23,962 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000003-wal.1732376783740.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:23,963 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000003-wal.1732376783740.temp to hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000020 2024-11-23T15:46:23,964 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 46 ms; skipped=2; WAL=hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-23T15:46:23,964 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740, journal: Splitting hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740, size=3.2 K (3256bytes) at 1732376783889Finishing writing output for hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740 so closing down at 1732376783920 (+31 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000003-wal.1732376783740.temp at 1732376783951 (+31 ms)3 split writer threads finished at 1732376783952 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000003-wal.1732376783740.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732376783962 (+10 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000003-wal.1732376783740.temp to hdfs://localhost:33345/hbase/data/default/testSequentialEditLogSeqNum/d549dc371d24b9be39236f3680b6fe8e/recovered.edits/0000000000000000020 at 1732376783963 (+1 ms)Processed 17 edits across 1 Regions in 46 ms; skipped=2; WAL=hdfs://localhost:33345/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732376783733/wal.1732376783740, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732376783964 (+1 ms) 2024-11-23T15:46:23,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=413 (was 408) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:33976 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:34000 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:55312 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 301), ProcessCount=11 (was 11), AvailableMemoryMB=7755 (was 7763) 2024-11-23T15:46:23,988 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=7755 2024-11-23T15:46:24,004 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:24,007 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:24,047 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:24,050 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44861388, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-44861388, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:24,063 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44861388/hregion-44861388.1732376784051, exclude list is [], retry=0 2024-11-23T15:46:24,067 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:24,067 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:24,067 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:24,094 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44861388/hregion-44861388.1732376784051 2024-11-23T15:46:24,094 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:24,094 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4c4c7be5b25a3872ef13b99011700737, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741879_1056 (size=70) 2024-11-23T15:46:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741879_1056 (size=70) 2024-11-23T15:46:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741879_1056 (size=70) 2024-11-23T15:46:24,105 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:24,107 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,108 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName a 2024-11-23T15:46:24,108 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:24,109 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:24,109 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,110 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName b 2024-11-23T15:46:24,110 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:24,111 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:24,111 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,112 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName c 2024-11-23T15:46:24,113 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:24,113 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:24,113 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,114 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,114 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,115 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,115 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,116 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:24,117 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,120 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:24,121 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4c4c7be5b25a3872ef13b99011700737; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65790378, jitterRate=-0.01964697241783142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:24,121 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4c4c7be5b25a3872ef13b99011700737: Writing region info on filesystem at 1732376784105Initializing all the Stores at 1732376784106 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376784106Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376784106Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376784106Cleaning up temporary data from old regions at 1732376784115 (+9 ms)Region opened successfully at 1732376784121 (+6 ms) 2024-11-23T15:46:24,122 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4c4c7be5b25a3872ef13b99011700737, disabling compactions & flushes 2024-11-23T15:46:24,122 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:24,122 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:24,122 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. after waiting 0 ms 2024-11-23T15:46:24,122 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:24,122 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:24,122 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4c4c7be5b25a3872ef13b99011700737: Waiting for close lock at 1732376784122Disabling compacts and flushes for region at 1732376784122Disabling writes for close at 1732376784122Writing region close event to WAL at 1732376784122Closed at 1732376784122 2024-11-23T15:46:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741878_1055 (size=175) 2024-11-23T15:46:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741878_1055 (size=175) 2024-11-23T15:46:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741878_1055 (size=175) 2024-11-23T15:46:24,130 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:24,130 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44861388:(num 1732376784051) 2024-11-23T15:46:24,130 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:24,133 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:24,146 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133, exclude list is [], retry=0 2024-11-23T15:46:24,149 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:24,150 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:24,150 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:24,176 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 2024-11-23T15:46:24,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:24,177 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c4c7be5b25a3872ef13b99011700737, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:24,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:24,177 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,177 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,179 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,180 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName a 2024-11-23T15:46:24,180 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:24,180 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:24,181 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,181 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName b 2024-11-23T15:46:24,182 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:24,182 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:24,182 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,183 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName c 2024-11-23T15:46:24,183 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:24,184 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:24,184 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,185 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,187 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,187 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,188 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:24,189 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:24,190 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4c4c7be5b25a3872ef13b99011700737; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67704149, jitterRate=0.008870437741279602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:24,191 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4c4c7be5b25a3872ef13b99011700737: Writing region info on filesystem at 1732376784177Initializing all the Stores at 1732376784178 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376784178Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376784179 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376784179Cleaning up temporary data from old regions at 1732376784187 (+8 ms)Region opened successfully at 1732376784191 (+4 ms) 2024-11-23T15:46:24,197 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732376784195/Put/seqid=0 2024-11-23T15:46:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741881_1058 (size=4826) 2024-11-23T15:46:24,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741881_1058 (size=4826) 2024-11-23T15:46:24,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741881_1058 (size=4826) 2024-11-23T15:46:24,205 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33345/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 4c4c7be5b25a3872ef13b99011700737/a 2024-11-23T15:46:24,212 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-23T15:46:24,212 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T15:46:24,213 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4c4c7be5b25a3872ef13b99011700737: 2024-11-23T15:46:24,214 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/a/861a0990e4c64600a35e88118fe87118_SeqId_3_ 2024-11-23T15:46:24,216 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33345/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4c4c7be5b25a3872ef13b99011700737/a as hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/a/861a0990e4c64600a35e88118fe87118_SeqId_3_ - updating store file list. 2024-11-23T15:46:24,222 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 861a0990e4c64600a35e88118fe87118_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:24,223 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/a/861a0990e4c64600a35e88118fe87118_SeqId_3_ into 4c4c7be5b25a3872ef13b99011700737/a 2024-11-23T15:46:24,223 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33345/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 4c4c7be5b25a3872ef13b99011700737/a (new location: hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/a/861a0990e4c64600a35e88118fe87118_SeqId_3_) 2024-11-23T15:46:24,279 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133, size=0 (0bytes) 2024-11-23T15:46:24,279 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 might be still open, length is 0 2024-11-23T15:46:24,279 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 2024-11-23T15:46:24,280 WARN [IPC Server handler 3 on default port 33345 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-23T15:46:24,280 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 after 1ms 2024-11-23T15:46:25,580 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:38858 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44917:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38858 dst: /127.0.0.1:44917 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:25,580 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:55398 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55398 dst: /127.0.0.1:35471 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:25,579 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:60486 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37735:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60486 dst: /127.0.0.1:37735 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37735 remote=/127.0.0.1:60486]. Total timeout mills is 60000, 58650 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:25,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741880_1059 (size=606) 2024-11-23T15:46:25,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741880_1059 (size=606) 2024-11-23T15:46:25,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741880_1059 (size=606) 2024-11-23T15:46:28,281 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 after 4002ms 2024-11-23T15:46:28,309 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:28,309 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 took 4030ms 2024-11-23T15:46:28,311 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133; continuing. 2024-11-23T15:46:28,311 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 so closing down 2024-11-23T15:46:28,311 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:28,313 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732376784133.temp 2024-11-23T15:46:28,336 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005-wal.1732376784133.temp 2024-11-23T15:46:28,337 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:28,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741882_1060 (size=329) 2024-11-23T15:46:28,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741882_1060 (size=329) 2024-11-23T15:46:28,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741882_1060 (size=329) 2024-11-23T15:46:28,345 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005-wal.1732376784133.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:28,346 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005-wal.1732376784133.temp to hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005 2024-11-23T15:46:28,347 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T15:46:28,347 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133, journal: Splitting hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133, size=0 (0bytes) at 1732376784279Finishing writing output for hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 so closing down at 1732376788311 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005-wal.1732376784133.temp at 1732376788336 (+25 ms)3 split writer threads finished at 1732376788337 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005-wal.1732376784133.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732376788345 (+8 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005-wal.1732376784133.temp to hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005 at 1732376788346 (+1 ms)Processed 2 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133, size=0, length=0, corrupted=false, cancelled=false at 1732376788347 (+1 ms) 2024-11-23T15:46:28,349 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376784133 2024-11-23T15:46:28,349 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005 2024-11-23T15:46:28,350 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:28,351 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:28,365 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376788352, exclude list is [], retry=0 2024-11-23T15:46:28,368 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:28,368 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:28,369 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:28,392 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376788352 2024-11-23T15:46:28,393 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:28,393 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c4c7be5b25a3872ef13b99011700737, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:28,393 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:28,393 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,393 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,395 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,396 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName a 2024-11-23T15:46:28,396 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:28,403 DEBUG [StoreFileOpener-4c4c7be5b25a3872ef13b99011700737-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 861a0990e4c64600a35e88118fe87118_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:28,403 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/a/861a0990e4c64600a35e88118fe87118_SeqId_3_ 2024-11-23T15:46:28,403 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:28,403 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,404 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName b 2024-11-23T15:46:28,404 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:28,405 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:28,405 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,406 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c4c7be5b25a3872ef13b99011700737 columnFamilyName c 2024-11-23T15:46:28,406 DEBUG [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:28,406 INFO [StoreOpener-4c4c7be5b25a3872ef13b99011700737-1 {}] regionserver.HStore(327): Store=4c4c7be5b25a3872ef13b99011700737/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:28,406 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,407 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,410 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005 2024-11-23T15:46:28,434 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:28,436 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005 2024-11-23T15:46:28,436 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4c4c7be5b25a3872ef13b99011700737 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-23T15:46:28,450 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/.tmp/a/c450f4faf8e04f77a635e2a7f61ebfe3 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732376784229/Put/seqid=0 2024-11-23T15:46:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741884_1062 (size=5149) 2024-11-23T15:46:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741884_1062 (size=5149) 2024-11-23T15:46:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741884_1062 (size=5149) 2024-11-23T15:46:28,458 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/.tmp/a/c450f4faf8e04f77a635e2a7f61ebfe3 2024-11-23T15:46:28,464 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/.tmp/a/c450f4faf8e04f77a635e2a7f61ebfe3 as hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/a/c450f4faf8e04f77a635e2a7f61ebfe3 2024-11-23T15:46:28,470 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/a/c450f4faf8e04f77a635e2a7f61ebfe3, entries=1, sequenceid=5, filesize=5.0 K 2024-11-23T15:46:28,470 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4c4c7be5b25a3872ef13b99011700737 in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-11-23T15:46:28,471 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/0000000000000000005 2024-11-23T15:46:28,472 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,472 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,473 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:28,475 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4c4c7be5b25a3872ef13b99011700737 2024-11-23T15:46:28,477 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/4c4c7be5b25a3872ef13b99011700737/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-23T15:46:28,477 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4c4c7be5b25a3872ef13b99011700737; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62520029, jitterRate=-0.06837897002696991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:28,478 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4c4c7be5b25a3872ef13b99011700737: Writing region info on filesystem at 1732376788394Initializing all the Stores at 1732376788395 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376788395Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376788395Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376788395Obtaining lock to block concurrent updates at 1732376788436 (+41 ms)Preparing flush snapshotting stores in 4c4c7be5b25a3872ef13b99011700737 at 1732376788436Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732376788436Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. at 1732376788436Flushing 4c4c7be5b25a3872ef13b99011700737/a: creating writer at 1732376788436Flushing 4c4c7be5b25a3872ef13b99011700737/a: appending metadata at 1732376788450 (+14 ms)Flushing 4c4c7be5b25a3872ef13b99011700737/a: closing flushed file at 1732376788450Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79e80699: reopening flushed file at 1732376788463 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 4c4c7be5b25a3872ef13b99011700737 in 34ms, sequenceid=5, compaction requested=false; wal=null at 1732376788470 (+7 ms)Cleaning up temporary data from old regions at 1732376788472 (+2 ms)Region opened successfully at 1732376788478 (+6 ms) 2024-11-23T15:46:28,482 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4c4c7be5b25a3872ef13b99011700737, disabling compactions & flushes 2024-11-23T15:46:28,482 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:28,482 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:28,482 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. after waiting 0 ms 2024-11-23T15:46:28,482 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:28,482 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732376784004.4c4c7be5b25a3872ef13b99011700737. 2024-11-23T15:46:28,482 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4c4c7be5b25a3872ef13b99011700737: Waiting for close lock at 1732376788481Disabling compacts and flushes for region at 1732376788481Disabling writes for close at 1732376788482 (+1 ms)Writing region close event to WAL at 1732376788482Closed at 1732376788482 2024-11-23T15:46:28,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741883_1061 (size=175) 2024-11-23T15:46:28,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741883_1061 (size=175) 2024-11-23T15:46:28,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741883_1061 (size=175) 2024-11-23T15:46:28,488 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:28,488 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732376788352) 2024-11-23T15:46:28,500 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=416 (was 413) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:43251 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713241193_22 at /127.0.0.1:38892 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33345 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43251 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:33345 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713241193_22 at /127.0.0.1:55444 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1713241193_22 at /127.0.0.1:60510 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=277 (was 301), ProcessCount=11 (was 11), AvailableMemoryMB=7773 (was 7755) - AvailableMemoryMB LEAK? - 2024-11-23T15:46:28,502 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-23T15:46:28,513 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=416, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=7773 2024-11-23T15:46:28,513 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-23T15:46:28,531 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:28,535 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:46:28,539 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,34021,1732376771486 2024-11-23T15:46:28,541 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f2c4182 2024-11-23T15:46:28,542 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:46:28,544 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:46:28,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:46:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-23T15:46:28,555 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:46:28,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-23T15:46:28,558 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:28,560 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:46:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:46:28,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741885_1063 (size=694) 2024-11-23T15:46:28,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741885_1063 (size=694) 2024-11-23T15:46:28,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741885_1063 (size=694) 2024-11-23T15:46:28,574 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c6b84da1fb3ebdb0344eecf253fc022, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748 2024-11-23T15:46:28,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741886_1064 (size=77) 2024-11-23T15:46:28,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741886_1064 (size=77) 2024-11-23T15:46:28,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741886_1064 (size=77) 2024-11-23T15:46:28,584 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:28,584 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 6c6b84da1fb3ebdb0344eecf253fc022, disabling compactions & flushes 2024-11-23T15:46:28,585 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:28,585 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:28,585 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. after waiting 0 ms 2024-11-23T15:46:28,585 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:28,585 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:28,585 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c6b84da1fb3ebdb0344eecf253fc022: Waiting for close lock at 1732376788584Disabling compacts and flushes for region at 1732376788584Disabling writes for close at 1732376788585 (+1 ms)Writing region close event to WAL at 1732376788585Closed at 1732376788585 2024-11-23T15:46:28,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:46:28,592 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732376788587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732376788587"}]},"ts":"1732376788587"} 2024-11-23T15:46:28,596 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:46:28,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:46:28,602 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376788598"}]},"ts":"1732376788598"} 2024-11-23T15:46:28,606 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-23T15:46:28,607 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {b712f9af2c12=0} racks are {/default-rack=0} 2024-11-23T15:46:28,608 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T15:46:28,608 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T15:46:28,608 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-23T15:46:28,608 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T15:46:28,608 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T15:46:28,608 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-23T15:46:28,608 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T15:46:28,609 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T15:46:28,609 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-23T15:46:28,609 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T15:46:28,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN}] 2024-11-23T15:46:28,613 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN 2024-11-23T15:46:28,614 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN; state=OFFLINE, location=b712f9af2c12,45181,1732376772204; forceNewPlan=false, retain=false 2024-11-23T15:46:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:46:28,766 INFO [b712f9af2c12:34021 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T15:46:28,767 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPENING, regionLocation=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:28,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN because future has completed 2024-11-23T15:46:28,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204}] 2024-11-23T15:46:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:46:28,933 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:28,934 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c6b84da1fb3ebdb0344eecf253fc022, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:28,934 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,935 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:28,935 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,935 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,937 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,939 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf1 2024-11-23T15:46:28,939 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:28,940 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:28,940 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,942 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf2 2024-11-23T15:46:28,942 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:28,943 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:28,943 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,944 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,944 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,945 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,945 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,946 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T15:46:28,948 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,950 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:28,951 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6c6b84da1fb3ebdb0344eecf253fc022; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72869383, jitterRate=0.0858384221792221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T15:46:28,951 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:28,951 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6c6b84da1fb3ebdb0344eecf253fc022: Running coprocessor pre-open hook at 1732376788935Writing region info on filesystem at 1732376788935Initializing all the Stores at 1732376788936 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376788936Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376788937 (+1 ms)Cleaning up temporary data from old regions at 1732376788945 (+8 ms)Running coprocessor post-open hooks at 1732376788951 (+6 ms)Region opened successfully at 1732376788951 2024-11-23T15:46:28,953 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., pid=6, masterSystemTime=1732376788927 2024-11-23T15:46:28,956 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:28,956 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:28,957 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:28,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204 because future has completed 2024-11-23T15:46:28,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T15:46:28,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204 in 190 msec 2024-11-23T15:46:28,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T15:46:28,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN in 355 msec 2024-11-23T15:46:28,969 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:46:28,970 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376788969"}]},"ts":"1732376788969"} 2024-11-23T15:46:28,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-23T15:46:28,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:46:28,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 424 msec 2024-11-23T15:46:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:46:29,199 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-23T15:46:29,199 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-23T15:46:29,202 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:46:29,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-23T15:46:29,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T15:46:29,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-23T15:46:29,224 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=2] 2024-11-23T15:46:29,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6c6b84da1fb3ebdb0344eecf253fc022, source=b712f9af2c12,45181,1732376772204, destination=b712f9af2c12,34281,1732376772315, warming up region on b712f9af2c12,34281,1732376772315 2024-11-23T15:46:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T15:46:29,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6c6b84da1fb3ebdb0344eecf253fc022, source=b712f9af2c12,45181,1732376772204, destination=b712f9af2c12,34281,1732376772315, running balancer 2024-11-23T15:46:29,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60387, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T15:46:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE 2024-11-23T15:46:29,247 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE 2024-11-23T15:46:29,249 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=CLOSING, regionLocation=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:29,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6c6b84da1fb3ebdb0344eecf253fc022, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:29,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE because future has completed 2024-11-23T15:46:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:29,252 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:46:29,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204}] 2024-11-23T15:46:29,252 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,254 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf1 2024-11-23T15:46:29,254 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:29,255 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:29,255 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,256 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf2 2024-11-23T15:46:29,256 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:29,257 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(1722): Closing 6c6b84da1fb3ebdb0344eecf253fc022, disabling compactions & flushes 2024-11-23T15:46:29,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. after waiting 0 ms 2024-11-23T15:46:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] regionserver.HRegion(1676): Region close journal for 6c6b84da1fb3ebdb0344eecf253fc022: Waiting for close lock at 1732376789257Disabling compacts and flushes for region at 1732376789257Disabling writes for close at 1732376789257Writing region close event to WAL at 1732376789258 (+1 ms)Closed at 1732376789258 2024-11-23T15:46:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-23T15:46:29,415 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,415 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:46:29,417 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 6c6b84da1fb3ebdb0344eecf253fc022, disabling compactions & flushes 2024-11-23T15:46:29,417 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,417 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,417 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. after waiting 0 ms 2024-11-23T15:46:29,417 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,418 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 6c6b84da1fb3ebdb0344eecf253fc022 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-23T15:46:29,438 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf1/6879621c1ee741c8941a7f02486231e8 is 35, key is r1/cf1:q/1732376789226/Put/seqid=0 2024-11-23T15:46:29,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741887_1065 (size=4783) 2024-11-23T15:46:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741887_1065 (size=4783) 2024-11-23T15:46:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741887_1065 (size=4783) 2024-11-23T15:46:29,447 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf1/6879621c1ee741c8941a7f02486231e8 2024-11-23T15:46:29,456 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf1/6879621c1ee741c8941a7f02486231e8 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8 2024-11-23T15:46:29,463 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8, entries=1, sequenceid=5, filesize=4.7 K 2024-11-23T15:46:29,465 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6c6b84da1fb3ebdb0344eecf253fc022 in 47ms, sequenceid=5, compaction requested=false 2024-11-23T15:46:29,465 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-23T15:46:29,470 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T15:46:29,473 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,473 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 6c6b84da1fb3ebdb0344eecf253fc022: Waiting for close lock at 1732376789416Running coprocessor pre-close hooks at 1732376789416Disabling compacts and flushes for region at 1732376789416Disabling writes for close at 1732376789417 (+1 ms)Obtaining lock to block concurrent updates at 1732376789418 (+1 ms)Preparing flush snapshotting stores in 6c6b84da1fb3ebdb0344eecf253fc022 at 1732376789418Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732376789419 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. at 1732376789421 (+2 ms)Flushing 6c6b84da1fb3ebdb0344eecf253fc022/cf1: creating writer at 1732376789421Flushing 6c6b84da1fb3ebdb0344eecf253fc022/cf1: appending metadata at 1732376789438 (+17 ms)Flushing 6c6b84da1fb3ebdb0344eecf253fc022/cf1: closing flushed file at 1732376789438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79bf9fa0: reopening flushed file at 1732376789454 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6c6b84da1fb3ebdb0344eecf253fc022 in 47ms, sequenceid=5, compaction requested=false at 1732376789465 (+11 ms)Writing region close event to WAL at 1732376789466 (+1 ms)Running coprocessor post-close hooks at 1732376789471 (+5 ms)Closed at 1732376789473 (+2 ms) 2024-11-23T15:46:29,474 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 6c6b84da1fb3ebdb0344eecf253fc022 move to b712f9af2c12,34281,1732376772315 record at close sequenceid=5 2024-11-23T15:46:29,477 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=CLOSED 2024-11-23T15:46:29,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204 because future has completed 2024-11-23T15:46:29,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T15:46:29,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204 in 230 msec 2024-11-23T15:46:29,487 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE; state=CLOSED, location=b712f9af2c12,34281,1732376772315; forceNewPlan=false, retain=false 2024-11-23T15:46:29,637 INFO [b712f9af2c12:34021 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T15:46:29,638 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPENING, regionLocation=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:29,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE because future has completed 2024-11-23T15:46:29,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315}] 2024-11-23T15:46:29,804 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,804 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c6b84da1fb3ebdb0344eecf253fc022, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:29,804 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,804 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:29,804 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,804 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,806 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,807 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf1 2024-11-23T15:46:29,807 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:29,814 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8 2024-11-23T15:46:29,814 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:29,814 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,815 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf2 2024-11-23T15:46:29,816 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:29,816 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:29,816 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,817 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,818 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,819 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,819 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,820 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T15:46:29,822 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,822 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 6c6b84da1fb3ebdb0344eecf253fc022; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64912946, jitterRate=-0.03272172808647156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T15:46:29,823 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:29,823 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6c6b84da1fb3ebdb0344eecf253fc022: Running coprocessor pre-open hook at 1732376789804Writing region info on filesystem at 1732376789804Initializing all the Stores at 1732376789805 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376789805Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376789806 (+1 ms)Cleaning up temporary data from old regions at 1732376789819 (+13 ms)Running coprocessor post-open hooks at 1732376789823 (+4 ms)Region opened successfully at 1732376789823 2024-11-23T15:46:29,824 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., pid=9, masterSystemTime=1732376789800 2024-11-23T15:46:29,827 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,827 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:29,828 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPEN, openSeqNum=9, regionLocation=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:29,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315 because future has completed 2024-11-23T15:46:29,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-23T15:46:29,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315 in 187 msec 2024-11-23T15:46:29,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE in 590 msec 2024-11-23T15:46:29,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T15:46:29,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T15:46:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:58250 deadline: 1732376849860, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=34281 startCode=1732376772315. As of locationSeqNum=5. 2024-11-23T15:46:29,883 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=34281 startCode=1732376772315. As of locationSeqNum=5. 2024-11-23T15:46:29,884 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=34281 startCode=1732376772315. As of locationSeqNum=5. 2024-11-23T15:46:29,884 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,34281,1732376772315, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=34281 startCode=1732376772315. As of locationSeqNum=5. 2024-11-23T15:46:29,997 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:46:30,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:46:30,011 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6c6b84da1fb3ebdb0344eecf253fc022 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-23T15:46:30,029 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf1/55227f9210fe4d048944ab5077406fd6 is 29, key is r1/cf1:/1732376790001/DeleteFamily/seqid=0 2024-11-23T15:46:30,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741888_1066 (size=4906) 2024-11-23T15:46:30,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741888_1066 (size=4906) 2024-11-23T15:46:30,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741888_1066 (size=4906) 2024-11-23T15:46:30,038 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf1/55227f9210fe4d048944ab5077406fd6 2024-11-23T15:46:30,044 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 55227f9210fe4d048944ab5077406fd6 2024-11-23T15:46:30,058 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf2/8d3bb27188704a28b577ca91f8a498c5 is 29, key is r1/cf2:/1732376790001/DeleteFamily/seqid=0 2024-11-23T15:46:30,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741889_1067 (size=4906) 2024-11-23T15:46:30,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741889_1067 (size=4906) 2024-11-23T15:46:30,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741889_1067 (size=4906) 2024-11-23T15:46:30,066 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf2/8d3bb27188704a28b577ca91f8a498c5 2024-11-23T15:46:30,072 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8d3bb27188704a28b577ca91f8a498c5 2024-11-23T15:46:30,073 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf1/55227f9210fe4d048944ab5077406fd6 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/55227f9210fe4d048944ab5077406fd6 2024-11-23T15:46:30,080 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 55227f9210fe4d048944ab5077406fd6 2024-11-23T15:46:30,081 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/55227f9210fe4d048944ab5077406fd6, entries=1, sequenceid=12, filesize=4.8 K 2024-11-23T15:46:30,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf2/8d3bb27188704a28b577ca91f8a498c5 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/8d3bb27188704a28b577ca91f8a498c5 2024-11-23T15:46:30,089 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8d3bb27188704a28b577ca91f8a498c5 2024-11-23T15:46:30,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/8d3bb27188704a28b577ca91f8a498c5, entries=1, sequenceid=12, filesize=4.8 K 2024-11-23T15:46:30,090 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 6c6b84da1fb3ebdb0344eecf253fc022 in 80ms, sequenceid=12, compaction requested=false 2024-11-23T15:46:30,090 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6c6b84da1fb3ebdb0344eecf253fc022: 2024-11-23T15:46:30,093 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T15:46:30,094 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6c6b84da1fb3ebdb0344eecf253fc022/cf1 is initiating major compaction (all files) 2024-11-23T15:46:30,095 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T15:46:30,095 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:30,095 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6c6b84da1fb3ebdb0344eecf253fc022/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,095 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8, hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/55227f9210fe4d048944ab5077406fd6] into tmpdir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp, totalSize=9.5 K 2024-11-23T15:46:30,097 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6879621c1ee741c8941a7f02486231e8, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732376789226 2024-11-23T15:46:30,097 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 55227f9210fe4d048944ab5077406fd6, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-23T15:46:30,110 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6c6b84da1fb3ebdb0344eecf253fc022#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:46:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741890_1068 (size=4626) 2024-11-23T15:46:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741890_1068 (size=4626) 2024-11-23T15:46:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741890_1068 (size=4626) 2024-11-23T15:46:30,126 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf1/a32d2eaf95d245abb2bcb54e473146e4 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/a32d2eaf95d245abb2bcb54e473146e4 2024-11-23T15:46:30,140 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 6c6b84da1fb3ebdb0344eecf253fc022/cf1 of 6c6b84da1fb3ebdb0344eecf253fc022 into a32d2eaf95d245abb2bcb54e473146e4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:46:30,140 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6c6b84da1fb3ebdb0344eecf253fc022: 2024-11-23T15:46:30,141 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-23T15:46:30,141 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6c6b84da1fb3ebdb0344eecf253fc022/cf2 is initiating major compaction (all files) 2024-11-23T15:46:30,141 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T15:46:30,141 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:46:30,141 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6c6b84da1fb3ebdb0344eecf253fc022/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,141 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/8d3bb27188704a28b577ca91f8a498c5] into tmpdir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp, totalSize=4.8 K 2024-11-23T15:46:30,142 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8d3bb27188704a28b577ca91f8a498c5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-23T15:46:30,148 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6c6b84da1fb3ebdb0344eecf253fc022#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:46:30,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741891_1069 (size=4592) 2024-11-23T15:46:30,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741891_1069 (size=4592) 2024-11-23T15:46:30,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741891_1069 (size=4592) 2024-11-23T15:46:30,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/.tmp/cf2/85e38a0390ff4296af839bdd52f58ef6 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/85e38a0390ff4296af839bdd52f58ef6 2024-11-23T15:46:30,171 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 6c6b84da1fb3ebdb0344eecf253fc022/cf2 of 6c6b84da1fb3ebdb0344eecf253fc022 into 85e38a0390ff4296af839bdd52f58ef6(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:46:30,171 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6c6b84da1fb3ebdb0344eecf253fc022: 2024-11-23T15:46:30,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6c6b84da1fb3ebdb0344eecf253fc022, source=b712f9af2c12,34281,1732376772315, destination=b712f9af2c12,45181,1732376772204, warming up region on b712f9af2c12,45181,1732376772204 2024-11-23T15:46:30,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6c6b84da1fb3ebdb0344eecf253fc022, source=b712f9af2c12,34281,1732376772315, destination=b712f9af2c12,45181,1732376772204, running balancer 2024-11-23T15:46:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE 2024-11-23T15:46:30,178 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE 2024-11-23T15:46:30,180 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=CLOSING, regionLocation=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:30,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6c6b84da1fb3ebdb0344eecf253fc022, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:30,181 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,183 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf1 2024-11-23T15:46:30,183 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:30,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE because future has completed 2024-11-23T15:46:30,188 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:46:30,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315}] 2024-11-23T15:46:30,196 INFO [StoreFileOpener-6c6b84da1fb3ebdb0344eecf253fc022-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 55227f9210fe4d048944ab5077406fd6 2024-11-23T15:46:30,196 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/55227f9210fe4d048944ab5077406fd6 2024-11-23T15:46:30,202 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8 2024-11-23T15:46:30,209 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/a32d2eaf95d245abb2bcb54e473146e4 2024-11-23T15:46:30,209 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:30,209 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,210 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf2 2024-11-23T15:46:30,210 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:30,216 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/85e38a0390ff4296af839bdd52f58ef6 2024-11-23T15:46:30,221 INFO [StoreFileOpener-6c6b84da1fb3ebdb0344eecf253fc022-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8d3bb27188704a28b577ca91f8a498c5 2024-11-23T15:46:30,222 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/8d3bb27188704a28b577ca91f8a498c5 2024-11-23T15:46:30,222 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(1722): Closing 6c6b84da1fb3ebdb0344eecf253fc022, disabling compactions & flushes 2024-11-23T15:46:30,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. after waiting 0 ms 2024-11-23T15:46:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45181 {}] regionserver.HRegion(1676): Region close journal for 6c6b84da1fb3ebdb0344eecf253fc022: Waiting for close lock at 1732376790222Disabling compacts and flushes for region at 1732376790222Disabling writes for close at 1732376790222Writing region close event to WAL at 1732376790223 (+1 ms)Closed at 1732376790223 2024-11-23T15:46:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-23T15:46:30,342 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,342 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:46:30,342 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 6c6b84da1fb3ebdb0344eecf253fc022, disabling compactions & flushes 2024-11-23T15:46:30,342 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,342 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,342 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. after waiting 0 ms 2024-11-23T15:46:30,342 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,343 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8, hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/55227f9210fe4d048944ab5077406fd6] to archive 2024-11-23T15:46:30,346 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:46:30,349 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8 to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/6879621c1ee741c8941a7f02486231e8 2024-11-23T15:46:30,351 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/55227f9210fe4d048944ab5077406fd6 to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/55227f9210fe4d048944ab5077406fd6 2024-11-23T15:46:30,362 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/8d3bb27188704a28b577ca91f8a498c5] to archive 2024-11-23T15:46:30,364 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:46:30,366 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/8d3bb27188704a28b577ca91f8a498c5 to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/8d3bb27188704a28b577ca91f8a498c5 2024-11-23T15:46:30,421 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-23T15:46:30,422 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,422 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 6c6b84da1fb3ebdb0344eecf253fc022: Waiting for close lock at 1732376790342Running coprocessor pre-close hooks at 1732376790342Disabling compacts and flushes for region at 1732376790342Disabling writes for close at 1732376790342Writing region close event to WAL at 1732376790367 (+25 ms)Running coprocessor post-close hooks at 1732376790422 (+55 ms)Closed at 1732376790422 2024-11-23T15:46:30,422 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 6c6b84da1fb3ebdb0344eecf253fc022 move to b712f9af2c12,45181,1732376772204 record at close sequenceid=12 2024-11-23T15:46:30,425 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,426 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=CLOSED 2024-11-23T15:46:30,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315 because future has completed 2024-11-23T15:46:30,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-23T15:46:30,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315 in 243 msec 2024-11-23T15:46:30,434 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE; state=CLOSED, location=b712f9af2c12,45181,1732376772204; forceNewPlan=false, retain=false 2024-11-23T15:46:30,585 INFO [b712f9af2c12:34021 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T15:46:30,585 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPENING, regionLocation=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:30,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE because future has completed 2024-11-23T15:46:30,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204}] 2024-11-23T15:46:30,747 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,748 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c6b84da1fb3ebdb0344eecf253fc022, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:30,748 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,749 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:30,749 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,749 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,752 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,753 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf1 2024-11-23T15:46:30,754 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:30,761 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/a32d2eaf95d245abb2bcb54e473146e4 2024-11-23T15:46:30,762 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:30,762 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,763 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf2 2024-11-23T15:46:30,763 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:30,771 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/85e38a0390ff4296af839bdd52f58ef6 2024-11-23T15:46:30,771 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:30,772 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,772 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,774 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,774 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,774 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,775 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T15:46:30,777 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,778 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 6c6b84da1fb3ebdb0344eecf253fc022; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66536052, jitterRate=-0.008535563945770264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T15:46:30,778 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,779 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 6c6b84da1fb3ebdb0344eecf253fc022: Running coprocessor pre-open hook at 1732376790749Writing region info on filesystem at 1732376790749Initializing all the Stores at 1732376790751 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376790751Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376790752 (+1 ms)Cleaning up temporary data from old regions at 1732376790774 (+22 ms)Running coprocessor post-open hooks at 1732376790778 (+4 ms)Region opened successfully at 1732376790779 (+1 ms) 2024-11-23T15:46:30,780 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., pid=12, masterSystemTime=1732376790741 2024-11-23T15:46:30,782 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,782 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,783 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPEN, openSeqNum=18, regionLocation=b712f9af2c12,45181,1732376772204 2024-11-23T15:46:30,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204 because future has completed 2024-11-23T15:46:30,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-23T15:46:30,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,45181,1732376772204 in 198 msec 2024-11-23T15:46:30,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, REOPEN/MOVE in 614 msec 2024-11-23T15:46:30,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T15:46:30,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T15:46:30,834 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b712f9af2c12,45181,1732376772204: testing ***** 2024-11-23T15:46:30,834 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-23T15:46:30,837 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-23T15:46:30,838 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-23T15:46:30,846 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-23T15:46:30,848 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-23T15:46:30,863 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 397704704 }, "NonHeapMemoryUsage": { "committed": 172228608, "init": 7667712, "max": -1, "used": 169553120 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b712f9af2c12", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 46, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 46, "ProcessCallTime_99th_percentile": 46, "ProcessCallTime_99.9th_percentile": 46, "ProcessCallTime_TimeRangeCount_0-1": 13, "ProcessCallTime_TimeRangeCount_1-3": 12, "ProcessCallTime_TimeRangeCount_3-10": 4, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 46, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 12, "TotalCallTime_98th_percentile": 46, "TotalCallTime_99th_percentile": 46, "TotalCallTime_99.9th_percentile": 46, "TotalCallTime_TimeRangeCount_0-1": 11, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5781 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b712f9af2c12", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:63612", "tag.serverName": "b712f9af2c12,34281,1732376772315", "tag.clusterId": "db9d882d-a2c9-4336-ae3a-b23aa9293c16", "tag.Context": "regionserver", "tag.Hostname": "b712f9af2c12", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732376772315, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 30, "MajorCompactionTime_max": 43, "MajorCompactionTime_mean": 36, "MajorCompactionTime_25th_percentile": 30, "MajorCompactionTime_median": 31, "MajorCompactionTime_75th_percentile": 43, "MajorCompactionTime_90th_percentile": 43, "MajorCompactionTime_95th_percentile": 43, "MajorCompactionTime_98th_percentile": 43, "MajorCompactionTime_99th_percentile": 43, "MajorCompactionTime_99.9th_percentile": 43, "MajorCompactionTime_TimeRangeCount_10-30": 1, "MajorCompactionTime_TimeRangeCount_30-100": 1, "ScanTime_num_ops": 1, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 3, "Delete_max": 3, "Delete_mean": 3, "Delete_25th_percentile": 3, "Delete_median": 3, "Delete_75th_percentile": 3, "Delete_90th_percentile": 3, "Delete_95th_percentile": 3, "Delete_98th_percentile": 3, "Delete_99th_percentile": 3, "Delete_99.9th_percentile": 3, "Delete_TimeRangeCount_1-3": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 30, "CompactionTime_max": 43, "CompactionTime_mean": 36, "CompactionTime_25th_percentile": 30, "CompactionTime_median": 31, "CompactionTime_75th_percentile": 43, "CompactionTime_90th_percentile": 43, "CompactionTime_95th_percentile": 43, "CompactionTime_98th_percentile": 43, "CompactionTime_99th_percentile": 43, "CompactionTime_99.9th_percentile": 43, "CompactionTime_TimeRangeCount_10-30": 1, "CompactionTime_TimeRangeCount_30-100": 1, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 80, "FlushTime_max": 80, "FlushTime_mean": 80, "FlushTime_25th_percentile": 80, "FlushTime_median": 80, "FlushTime_75th_percentile": 80, "FlushTime_90th_percentile": 80, "FlushTime_95th_percentile": 80, "FlushTime_98th_percentile": 80, "FlushTime_99th_percentile": 80, "FlushTime_99.9th_percentile": 80, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-23T15:46:30,867 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34021 {}] master.MasterRpcServices(700): b712f9af2c12,45181,1732376772204 reported a fatal error: ***** ABORTING region server b712f9af2c12,45181,1732376772204: testing ***** 2024-11-23T15:46:30,869 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b712f9af2c12,45181,1732376772204' ***** 2024-11-23T15:46:30,869 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-23T15:46:30,870 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T15:46:30,870 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T15:46:30,870 INFO [RS:0;b712f9af2c12:45181 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-23T15:46:30,870 INFO [RS:0;b712f9af2c12:45181 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-23T15:46:30,870 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(3091): Received CLOSE for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,871 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(956): aborting server b712f9af2c12,45181,1732376772204 2024-11-23T15:46:30,871 INFO [RS:0;b712f9af2c12:45181 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:46:30,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34281 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:43350 deadline: 1732376850871, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=45181 startCode=1732376772204. As of locationSeqNum=12. 2024-11-23T15:46:30,871 INFO [RS:0;b712f9af2c12:45181 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b712f9af2c12:45181. 2024-11-23T15:46:30,871 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6c6b84da1fb3ebdb0344eecf253fc022, disabling compactions & flushes 2024-11-23T15:46:30,871 DEBUG [RS:0;b712f9af2c12:45181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:46:30,871 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,871 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,871 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. after waiting 0 ms 2024-11-23T15:46:30,872 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,872 DEBUG [RS:0;b712f9af2c12:45181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:30,872 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,34281,1732376772315, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,34281,1732376772315, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=45181 startCode=1732376772204. As of locationSeqNum=12. 2024-11-23T15:46:30,872 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,34281,1732376772315, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=45181 startCode=1732376772204. As of locationSeqNum=12. 2024-11-23T15:46:30,872 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,34281,1732376772315, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b712f9af2c12 port=45181 startCode=1732376772204. As of locationSeqNum=12. 2024-11-23T15:46:30,872 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T15:46:30,872 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T15:46:30,872 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T15:46:30,872 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T15:46:30,873 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T15:46:30,873 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T15:46:30,873 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6c6b84da1fb3ebdb0344eecf253fc022=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.} 2024-11-23T15:46:30,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b712f9af2c12,45181,1732376772204 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:30,873 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T15:46:30,873 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T15:46:30,874 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T15:46:30,874 DEBUG [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:30,874 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T15:46:30,874 ERROR [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-23T15:46:30,874 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,875 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:46:30,875 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6c6b84da1fb3ebdb0344eecf253fc022: Waiting for close lock at 1732376790871Running coprocessor pre-close hooks at 1732376790871Disabling compacts and flushes for region at 1732376790871Disabling writes for close at 1732376790871Writing region close event to WAL at 1732376790874 (+3 ms)Running coprocessor post-close hooks at 1732376790874Closed at 1732376790874 2024-11-23T15:46:30,875 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b712f9af2c12,45181,1732376772204 aborting 2024-11-23T15:46:30,875 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T15:46:30,875 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b712f9af2c12,45181,1732376772204 aborting 2024-11-23T15:46:30,875 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:30,875 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732376790873Running coprocessor pre-close hooks at 1732376790873Disabling compacts and flushes for region at 1732376790873Disabling writes for close at 1732376790874 (+1 ms)Writing region close event to WAL at 1732376790874Running coprocessor post-close hooks at 1732376790874Closed at 1732376790875 (+1 ms) 2024-11-23T15:46:30,875 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 from cache 2024-11-23T15:46:30,875 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T15:46:30,950 INFO [regionserver/b712f9af2c12:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:46:30,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:30,979 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1] 2024-11-23T15:46:30,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b712f9af2c12,45181,1732376772204 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:30,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b712f9af2c12,45181,1732376772204 aborting 2024-11-23T15:46:30,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b712f9af2c12,45181,1732376772204 aborting 2024-11-23T15:46:30,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 from cache 2024-11-23T15:46:31,074 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(976): stopping server b712f9af2c12,45181,1732376772204; all regions closed. 2024-11-23T15:46:31,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741836_1012 (size=7323) 2024-11-23T15:46:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741836_1012 (size=7323) 2024-11-23T15:46:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741836_1012 (size=7323) 2024-11-23T15:46:31,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741833_1009 (size=2288) 2024-11-23T15:46:31,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741833_1009 (size=2288) 2024-11-23T15:46:31,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741833_1009 (size=2288) 2024-11-23T15:46:31,087 DEBUG [RS:0;b712f9af2c12:45181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:31,087 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:46:31,087 INFO [RS:0;b712f9af2c12:45181 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:46:31,088 INFO [RS:0;b712f9af2c12:45181 {}] hbase.ChoreService(370): Chore service for: regionserver/b712f9af2c12:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T15:46:31,088 INFO [RS:0;b712f9af2c12:45181 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:46:31,088 INFO [regionserver/b712f9af2c12:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:46:31,088 INFO [RS:0;b712f9af2c12:45181 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45181 2024-11-23T15:46:31,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:46:31,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b712f9af2c12,45181,1732376772204 2024-11-23T15:46:31,144 INFO [RS:0;b712f9af2c12:45181 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:46:31,153 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b712f9af2c12,45181,1732376772204] 2024-11-23T15:46:31,161 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b712f9af2c12,45181,1732376772204 already deleted, retry=false 2024-11-23T15:46:31,161 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b712f9af2c12,45181,1732376772204 on b712f9af2c12,34021,1732376771486 2024-11-23T15:46:31,167 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b712f9af2c12,45181,1732376772204, splitWal=true, meta=true 2024-11-23T15:46:31,169 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for b712f9af2c12,45181,1732376772204 (carryingMeta=true) b712f9af2c12,45181,1732376772204/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@79e3d41a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-23T15:46:31,170 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b712f9af2c12,45181,1732376772204, splitWal=true, meta=true 2024-11-23T15:46:31,172 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure b712f9af2c12,45181,1732376772204, splitWal=true, meta=true, isMeta: true 2024-11-23T15:46:31,173 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting 2024-11-23T15:46:31,174 INFO [PEWorker-1 {}] master.SplitWALManager(105): b712f9af2c12,45181,1732376772204 WAL count=1, meta=true 2024-11-23T15:46:31,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta}] 2024-11-23T15:46:31,183 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=b712f9af2c12,40435,1732376772380 2024-11-23T15:46:31,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, worker=b712f9af2c12,40435,1732376772380}] 2024-11-23T15:46:31,187 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:31,188 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1] 2024-11-23T15:46:31,190 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b712f9af2c12:45181 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b712f9af2c12/172.17.0.2:45181 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:31,191 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1, error=java.net.ConnectException: Call to address=b712f9af2c12:45181 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b712f9af2c12/172.17.0.2:45181 2024-11-23T15:46:31,191 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-23T15:46:31,191 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 from cache 2024-11-23T15:46:31,191 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address b712f9af2c12:45181 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b712f9af2c12/172.17.0.2:45181 2024-11-23T15:46:31,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:31,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45181-0x10169a94b600001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:31,254 INFO [RS:0;b712f9af2c12:45181 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:46:31,254 INFO [RS:0;b712f9af2c12:45181 {}] regionserver.HRegionServer(1031): Exiting; stopping=b712f9af2c12,45181,1732376772204; zookeeper connection closed. 2024-11-23T15:46:31,254 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@458818c1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@458818c1 2024-11-23T15:46:31,346 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T15:46:31,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40931, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T15:46:31,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40435 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-23T15:46:31,365 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, size=7.2 K (7323bytes) 2024-11-23T15:46:31,365 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta 2024-11-23T15:46:31,366 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta after 1ms 2024-11-23T15:46:31,391 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:31,391 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta took 26ms 2024-11-23T15:46:31,398 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-23T15:46:31,398 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta so closing down 2024-11-23T15:46:31,398 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:31,400 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta.temp 2024-11-23T15:46:31,424 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta.temp 2024-11-23T15:46:31,425 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741892_1070 (size=7071) 2024-11-23T15:46:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741892_1070 (size=7071) 2024-11-23T15:46:31,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741892_1070 (size=7071) 2024-11-23T15:46:31,432 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:31,434 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta.temp to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T15:46:31,434 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 43 ms; skipped=1; WAL=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-23T15:46:31,434 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, journal: Splitting hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, size=7.2 K (7323bytes) at 1732376791365Finishing writing output for hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta so closing down at 1732376791398 (+33 ms)Creating recovered edits writer path=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta.temp at 1732376791424 (+26 ms)3 split writer threads finished at 1732376791425 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732376791432 (+7 ms)Rename recovered edits hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta.temp to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732376791434 (+2 ms)Processed 16 edits across 1 Regions in 43 ms; skipped=1; WAL=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1732376791434 2024-11-23T15:46:31,434 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta 2024-11-23T15:46:31,436 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-23T15:46:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-23T15:46:31,443 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs 2024-11-23T15:46:31,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-23T15:46:31,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, worker=b712f9af2c12,40435,1732376772380 in 258 msec 2024-11-23T15:46:31,448 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=b712f9af2c12,40435,1732376772380 2024-11-23T15:46:31,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T15:46:31,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b712f9af2c12%2C45181%2C1732376772204.meta.1732376774424.meta, worker=b712f9af2c12,40435,1732376772380 in 273 msec 2024-11-23T15:46:31,454 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting dir is empty, no logs to split. 2024-11-23T15:46:31,454 INFO [PEWorker-1 {}] master.SplitWALManager(105): b712f9af2c12,45181,1732376772204 WAL count=0, meta=true 2024-11-23T15:46:31,455 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b712f9af2c12,45181,1732376772204 WAL splitting is done? wals=0, meta=true 2024-11-23T15:46:31,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T15:46:31,458 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:46:31,459 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T15:46:31,497 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:31,498 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1] 2024-11-23T15:46:31,499 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to b712f9af2c12:45181 this server is in the failed servers list 2024-11-23T15:46:31,500 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b712f9af2c12:45181 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b712f9af2c12:45181 2024-11-23T15:46:31,500 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b712f9af2c12:45181 2024-11-23T15:46:31,500 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 from cache 2024-11-23T15:46:31,610 DEBUG [b712f9af2c12:34021 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T15:46:31,610 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(204): Hosts are {b712f9af2c12=0} racks are {/default-rack=0} 2024-11-23T15:46:31,610 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T15:46:31,610 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T15:46:31,610 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T15:46:31,610 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T15:46:31,610 INFO [b712f9af2c12:34021 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T15:46:31,610 INFO [b712f9af2c12:34021 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T15:46:31,610 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T15:46:31,611 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b712f9af2c12,40435,1732376772380 2024-11-23T15:46:31,612 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,40435,1732376772380, state=OPENING 2024-11-23T15:46:31,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:31,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:31,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:31,644 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:46:31,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:31,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:31,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:31,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,40435,1732376772380}] 2024-11-23T15:46:31,802 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T15:46:31,802 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:31,803 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T15:46:31,806 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C40435%2C1732376772380.meta, suffix=.meta, logDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380, archiveDir=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs, maxLogs=32 2024-11-23T15:46:31,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-23T15:46:31,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:31,818 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-23T15:46:31,819 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:31,820 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:46:31,821 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T15:46:31,825 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380/b712f9af2c12%2C40435%2C1732376772380.meta.1732376791807.meta, exclude list is [], retry=0 2024-11-23T15:46:31,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:31,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:31,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:31,854 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380/b712f9af2c12%2C40435%2C1732376772380.meta.1732376791807.meta 2024-11-23T15:46:31,854 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:37309:37309)] 2024-11-23T15:46:31,855 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:31,855 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T15:46:31,855 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T15:46:31,855 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T15:46:31,855 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T15:46:31,855 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:31,856 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T15:46:31,856 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T15:46:31,857 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T15:46:31,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T15:46:31,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:31,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:31,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T15:46:31,860 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T15:46:31,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:31,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:31,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T15:46:31,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T15:46:31,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:31,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:31,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T15:46:31,863 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T15:46:31,863 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:31,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:46:31,864 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T15:46:31,865 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740 2024-11-23T15:46:31,866 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740 2024-11-23T15:46:31,867 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T15:46:31,892 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:31,895 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T15:46:31,895 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-23T15:46:31,915 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/info/93c5752550b046c19d005337c7e11638 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022./info:regioninfo/1732376790783/Put/seqid=0 2024-11-23T15:46:31,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741894_1072 (size=11177) 2024-11-23T15:46:31,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741894_1072 (size=11177) 2024-11-23T15:46:31,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741894_1072 (size=11177) 2024-11-23T15:46:31,926 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/info/93c5752550b046c19d005337c7e11638 2024-11-23T15:46:31,946 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/ns/b9eb5f0ef5314e74bde643c3e73df06e is 43, key is default/ns:d/1732376774633/Put/seqid=0 2024-11-23T15:46:31,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741895_1073 (size=5153) 2024-11-23T15:46:31,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741895_1073 (size=5153) 2024-11-23T15:46:31,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741895_1073 (size=5153) 2024-11-23T15:46:31,954 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/ns/b9eb5f0ef5314e74bde643c3e73df06e 2024-11-23T15:46:31,975 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/table/1765efcd96a24aff8003030207799a33 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732376788969/Put/seqid=0 2024-11-23T15:46:31,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741896_1074 (size=5431) 2024-11-23T15:46:31,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741896_1074 (size=5431) 2024-11-23T15:46:31,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741896_1074 (size=5431) 2024-11-23T15:46:31,983 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/table/1765efcd96a24aff8003030207799a33 2024-11-23T15:46:31,990 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/info/93c5752550b046c19d005337c7e11638 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/info/93c5752550b046c19d005337c7e11638 2024-11-23T15:46:31,996 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/info/93c5752550b046c19d005337c7e11638, entries=36, sequenceid=18, filesize=10.9 K 2024-11-23T15:46:31,997 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/ns/b9eb5f0ef5314e74bde643c3e73df06e as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/ns/b9eb5f0ef5314e74bde643c3e73df06e 2024-11-23T15:46:32,004 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/ns/b9eb5f0ef5314e74bde643c3e73df06e, entries=2, sequenceid=18, filesize=5.0 K 2024-11-23T15:46:32,005 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/table/1765efcd96a24aff8003030207799a33 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/table/1765efcd96a24aff8003030207799a33 2024-11-23T15:46:32,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:32,009 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,40435,1732376772380, seqNum=-1] 2024-11-23T15:46:32,010 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:46:32,012 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:46:32,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40435 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:57818 deadline: 1732376852012, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on b712f9af2c12,40435,1732376772380 2024-11-23T15:46:32,014 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,40435,1732376772380, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,40435,1732376772380, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on b712f9af2c12,40435,1732376772380 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:46:32,014 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,40435,1732376772380, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on b712f9af2c12,40435,1732376772380 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:46:32,014 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,40435,1732376772380, seqNum=-1 from cache 2024-11-23T15:46:32,014 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/table/1765efcd96a24aff8003030207799a33, entries=2, sequenceid=18, filesize=5.3 K 2024-11-23T15:46:32,015 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=18, compaction requested=false; wal=null 2024-11-23T15:46:32,016 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-23T15:46:32,018 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T15:46:32,018 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T15:46:32,019 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:46:32,021 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T15:46:32,024 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-23T15:46:32,025 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61200151, jitterRate=-0.08804668486118317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:46:32,025 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T15:46:32,026 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732376791856Writing region info on filesystem at 1732376791856Initializing all the Stores at 1732376791857 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376791857Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376791857Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376791857Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376791857Obtaining lock to block concurrent updates at 1732376791895 (+38 ms)Preparing flush snapshotting stores in 1588230740 at 1732376791895Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732376791895Flushing stores of hbase:meta,,1.1588230740 at 1732376791895Flushing 1588230740/info: creating writer at 1732376791895Flushing 1588230740/info: appending metadata at 1732376791914 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732376791914Flushing 1588230740/ns: creating writer at 1732376791932 (+18 ms)Flushing 1588230740/ns: appending metadata at 1732376791946 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732376791946Flushing 1588230740/table: creating writer at 1732376791960 (+14 ms)Flushing 1588230740/table: appending metadata at 1732376791975 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732376791975Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aba3584: reopening flushed file at 1732376791989 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@647157f8: reopening flushed file at 1732376791996 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46cf0ecb: reopening flushed file at 1732376792005 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=18, compaction requested=false; wal=null at 1732376792015 (+10 ms)Cleaning up temporary data from old regions at 1732376792018 (+3 ms)Running coprocessor post-open hooks at 1732376792025 (+7 ms)Region opened successfully at 1732376792026 (+1 ms) 2024-11-23T15:46:32,027 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732376791797 2024-11-23T15:46:32,030 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T15:46:32,030 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T15:46:32,031 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=b712f9af2c12,40435,1732376772380 2024-11-23T15:46:32,032 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,40435,1732376772380, state=OPEN 2024-11-23T15:46:32,065 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T15:46:32,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:32,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:32,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:46:32,069 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b712f9af2c12,40435,1732376772380 2024-11-23T15:46:32,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:32,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:32,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:46:32,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-23T15:46:32,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,40435,1732376772380 in 425 msec 2024-11-23T15:46:32,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-23T15:46:32,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 615 msec 2024-11-23T15:46:32,075 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): b712f9af2c12,45181,1732376772204 had 2 regions 2024-11-23T15:46:32,076 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b712f9af2c12,45181,1732376772204, splitWal=true, meta=true, isMeta: false 2024-11-23T15:46:32,078 INFO [PEWorker-2 {}] master.SplitWALManager(105): b712f9af2c12,45181,1732376772204 WAL count=1, meta=false 2024-11-23T15:46:32,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b712f9af2c12%2C45181%2C1732376772204.1732376773966}] 2024-11-23T15:46:32,081 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:32,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b712f9af2c12%2C45181%2C1732376772204.1732376773966, worker=b712f9af2c12,34281,1732376772315}] 2024-11-23T15:46:32,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34281 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-23T15:46:32,253 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966, size=2.2 K (2288bytes) 2024-11-23T15:46:32,253 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966 2024-11-23T15:46:32,253 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966 after 0ms 2024-11-23T15:46:32,278 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:32,278 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966 took 25ms 2024-11-23T15:46:32,281 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 6c6b84da1fb3ebdb0344eecf253fc022: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-23T15:46:32,281 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966 so closing down 2024-11-23T15:46:32,281 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:32,281 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:32,282 INFO [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-23T15:46:32,282 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966, journal: Splitting hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966, size=2.2 K (2288bytes) at 1732376792253Finishing writing output for hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966 so closing down at 1732376792281 (+28 ms)3 split writer threads finished at 1732376792281Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732376792282 (+1 ms) 2024-11-23T15:46:32,282 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966 2024-11-23T15:46:32,282 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-23T15:46:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34021 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-23T15:46:32,286 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting/b712f9af2c12%2C45181%2C1732376772204.1732376773966 to hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs 2024-11-23T15:46:32,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-23T15:46:32,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b712f9af2c12%2C45181%2C1732376772204.1732376773966, worker=b712f9af2c12,34281,1732376772315 in 205 msec 2024-11-23T15:46:32,289 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:32,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-23T15:46:32,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b712f9af2c12%2C45181%2C1732376772204.1732376773966, worker=b712f9af2c12,34281,1732376772315 in 211 msec 2024-11-23T15:46:32,293 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,45181,1732376772204-splitting dir is empty, no logs to split. 2024-11-23T15:46:32,293 INFO [PEWorker-2 {}] master.SplitWALManager(105): b712f9af2c12,45181,1732376772204 WAL count=0, meta=false 2024-11-23T15:46:32,293 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if b712f9af2c12,45181,1732376772204 WAL splitting is done? wals=0, meta=false 2024-11-23T15:46:32,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN}] 2024-11-23T15:46:32,297 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN 2024-11-23T15:46:32,298 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T15:46:32,449 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(204): Hosts are {b712f9af2c12=0} racks are {/default-rack=0} 2024-11-23T15:46:32,450 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T15:46:32,450 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T15:46:32,450 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T15:46:32,450 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T15:46:32,451 INFO [b712f9af2c12:34021 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T15:46:32,451 INFO [b712f9af2c12:34021 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T15:46:32,451 DEBUG [b712f9af2c12:34021 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T15:46:32,452 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPENING, regionLocation=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:32,455 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b712f9af2c12:45181 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b712f9af2c12/172.17.0.2:45181 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:32,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1, error=java.net.ConnectException: Call to address=b712f9af2c12:45181 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b712f9af2c12/172.17.0.2:45181 2024-11-23T15:46:32,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-23T15:46:32,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,45181,1732376772204, seqNum=-1 from cache 2024-11-23T15:46:32,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address b712f9af2c12:45181 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b712f9af2c12/172.17.0.2:45181 2024-11-23T15:46:32,568 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:32,568 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,40435,1732376772380, seqNum=-1] 2024-11-23T15:46:32,569 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:46:32,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49385, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:46:32,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN because future has completed 2024-11-23T15:46:32,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315}] 2024-11-23T15:46:32,739 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:32,739 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c6b84da1fb3ebdb0344eecf253fc022, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:32,740 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,740 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:32,740 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,740 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,742 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,743 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf1 2024-11-23T15:46:32,743 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:32,750 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf1/a32d2eaf95d245abb2bcb54e473146e4 2024-11-23T15:46:32,751 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:32,751 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,752 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6b84da1fb3ebdb0344eecf253fc022 columnFamilyName cf2 2024-11-23T15:46:32,752 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:32,758 DEBUG [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/cf2/85e38a0390ff4296af839bdd52f58ef6 2024-11-23T15:46:32,759 INFO [StoreOpener-6c6b84da1fb3ebdb0344eecf253fc022-1 {}] regionserver.HStore(327): Store=6c6b84da1fb3ebdb0344eecf253fc022/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:32,759 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,760 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,761 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,761 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,761 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,762 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-23T15:46:32,763 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,764 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 6c6b84da1fb3ebdb0344eecf253fc022; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63060889, jitterRate=-0.060319527983665466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-23T15:46:32,764 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:32,765 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 6c6b84da1fb3ebdb0344eecf253fc022: Running coprocessor pre-open hook at 1732376792740Writing region info on filesystem at 1732376792740Initializing all the Stores at 1732376792742 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376792742Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376792742Cleaning up temporary data from old regions at 1732376792761 (+19 ms)Running coprocessor post-open hooks at 1732376792764 (+3 ms)Region opened successfully at 1732376792765 (+1 ms) 2024-11-23T15:46:32,766 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., pid=21, masterSystemTime=1732376792731 2024-11-23T15:46:32,768 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:32,768 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:32,768 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6c6b84da1fb3ebdb0344eecf253fc022, regionState=OPEN, openSeqNum=18, regionLocation=b712f9af2c12,34281,1732376772315 2024-11-23T15:46:32,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315 because future has completed 2024-11-23T15:46:32,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-23T15:46:32,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c6b84da1fb3ebdb0344eecf253fc022, server=b712f9af2c12,34281,1732376772315 in 195 msec 2024-11-23T15:46:32,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-23T15:46:32,777 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server b712f9af2c12,45181,1732376772204 after splitting done 2024-11-23T15:46:32,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6c6b84da1fb3ebdb0344eecf253fc022, ASSIGN in 479 msec 2024-11-23T15:46:32,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure b712f9af2c12,45181,1732376772204, splitWal=true, meta=true in 1.6160 sec 2024-11-23T15:46:33,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:46:33,029 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,40435,1732376772380, seqNum=-1] 2024-11-23T15:46:33,147 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to b712f9af2c12:45181 this server is in the failed servers list 2024-11-23T15:46:33,148 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b712f9af2c12:45181 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b712f9af2c12:45181 2024-11-23T15:46:33,148 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b712f9af2c12:45181 2024-11-23T15:46:33,148 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,45181,1732376772204, seqNum=12 from cache 2024-11-23T15:46:33,368 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022., hostname=b712f9af2c12,34281,1732376772315, seqNum=18] 2024-11-23T15:46:33,381 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 416), OpenFileDescriptor=1050 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 277) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7696 (was 7773) 2024-11-23T15:46:33,381 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-23T15:46:33,393 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=7695 2024-11-23T15:46:33,393 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-23T15:46:33,406 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:33,408 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:33,409 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:33,411 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33819538, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-33819538, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:33,423 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33819538/hregion-33819538.1732376793412, exclude list is [], retry=0 2024-11-23T15:46:33,425 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:33,425 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:33,426 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:33,449 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33819538/hregion-33819538.1732376793412 2024-11-23T15:46:33,450 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:33,450 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ef896d7e639d709c75b185a128fae935, NAME => 'testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:33,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741898_1076 (size=67) 2024-11-23T15:46:33,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741898_1076 (size=67) 2024-11-23T15:46:33,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741898_1076 (size=67) 2024-11-23T15:46:33,459 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:33,460 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,462 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName a 2024-11-23T15:46:33,462 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,462 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,462 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,463 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName b 2024-11-23T15:46:33,463 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,464 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,464 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,464 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName c 2024-11-23T15:46:33,465 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,465 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,465 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,466 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,466 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,467 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,467 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,467 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:33,468 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,470 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:33,471 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ef896d7e639d709c75b185a128fae935; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70829262, jitterRate=0.05543825030326843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:33,472 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ef896d7e639d709c75b185a128fae935: Writing region info on filesystem at 1732376793459Initializing all the Stores at 1732376793460 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793460Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793460Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793460Cleaning up temporary data from old regions at 1732376793467 (+7 ms)Region opened successfully at 1732376793471 (+4 ms) 2024-11-23T15:46:33,472 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ef896d7e639d709c75b185a128fae935, disabling compactions & flushes 2024-11-23T15:46:33,472 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,472 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,472 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. after waiting 0 ms 2024-11-23T15:46:33,472 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,472 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,472 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ef896d7e639d709c75b185a128fae935: Waiting for close lock at 1732376793472Disabling compacts and flushes for region at 1732376793472Disabling writes for close at 1732376793472Writing region close event to WAL at 1732376793472Closed at 1732376793472 2024-11-23T15:46:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741897_1075 (size=175) 2024-11-23T15:46:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741897_1075 (size=175) 2024-11-23T15:46:33,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741897_1075 (size=175) 2024-11-23T15:46:33,477 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:33,477 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33819538:(num 1732376793412) 2024-11-23T15:46:33,477 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:33,479 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:33,491 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479, exclude list is [], retry=0 2024-11-23T15:46:33,493 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:33,493 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:33,494 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:33,519 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479 2024-11-23T15:46:33,519 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:33,519 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ef896d7e639d709c75b185a128fae935, NAME => 'testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:33,519 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:33,520 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,520 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,521 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,522 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName a 2024-11-23T15:46:33,522 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,522 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,522 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,523 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName b 2024-11-23T15:46:33,523 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,523 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,523 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,524 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName c 2024-11-23T15:46:33,524 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,524 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,525 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,525 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,526 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,527 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,527 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,528 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:33,529 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,530 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ef896d7e639d709c75b185a128fae935; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66551414, jitterRate=-0.008306652307510376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:33,531 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ef896d7e639d709c75b185a128fae935: Writing region info on filesystem at 1732376793520Initializing all the Stores at 1732376793520Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793520Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793521 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793521Cleaning up temporary data from old regions at 1732376793527 (+6 ms)Region opened successfully at 1732376793531 (+4 ms) 2024-11-23T15:46:33,555 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ef896d7e639d709c75b185a128fae935 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-23T15:46:33,574 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/a/034634dbc0ab40c893ddeb452a84eb79 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732376793531/Put/seqid=0 2024-11-23T15:46:33,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741900_1078 (size=5958) 2024-11-23T15:46:33,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741900_1078 (size=5958) 2024-11-23T15:46:33,581 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/a/034634dbc0ab40c893ddeb452a84eb79 2024-11-23T15:46:33,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741900_1078 (size=5958) 2024-11-23T15:46:33,598 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/b/6d03770f60004b2b945be3fee2dc3ae4 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732376793540/Put/seqid=0 2024-11-23T15:46:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741901_1079 (size=5958) 2024-11-23T15:46:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741901_1079 (size=5958) 2024-11-23T15:46:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741901_1079 (size=5958) 2024-11-23T15:46:33,605 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/b/6d03770f60004b2b945be3fee2dc3ae4 2024-11-23T15:46:33,624 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/c/927566bce01546448e0003ffedc2e9f2 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732376793547/Put/seqid=0 2024-11-23T15:46:33,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741902_1080 (size=5958) 2024-11-23T15:46:33,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741902_1080 (size=5958) 2024-11-23T15:46:33,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741902_1080 (size=5958) 2024-11-23T15:46:33,631 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/c/927566bce01546448e0003ffedc2e9f2 2024-11-23T15:46:33,636 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/a/034634dbc0ab40c893ddeb452a84eb79 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/a/034634dbc0ab40c893ddeb452a84eb79 2024-11-23T15:46:33,642 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/a/034634dbc0ab40c893ddeb452a84eb79, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T15:46:33,643 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/b/6d03770f60004b2b945be3fee2dc3ae4 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/b/6d03770f60004b2b945be3fee2dc3ae4 2024-11-23T15:46:33,648 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/b/6d03770f60004b2b945be3fee2dc3ae4, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T15:46:33,649 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/c/927566bce01546448e0003ffedc2e9f2 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/c/927566bce01546448e0003ffedc2e9f2 2024-11-23T15:46:33,655 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/c/927566bce01546448e0003ffedc2e9f2, entries=10, sequenceid=33, filesize=5.8 K 2024-11-23T15:46:33,656 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ef896d7e639d709c75b185a128fae935 in 102ms, sequenceid=33, compaction requested=false 2024-11-23T15:46:33,656 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ef896d7e639d709c75b185a128fae935: 2024-11-23T15:46:33,656 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ef896d7e639d709c75b185a128fae935, disabling compactions & flushes 2024-11-23T15:46:33,656 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,656 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,656 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. after waiting 0 ms 2024-11-23T15:46:33,656 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,658 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. 2024-11-23T15:46:33,658 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ef896d7e639d709c75b185a128fae935: Waiting for close lock at 1732376793656Disabling compacts and flushes for region at 1732376793656Disabling writes for close at 1732376793656Writing region close event to WAL at 1732376793658 (+2 ms)Closed at 1732376793658 2024-11-23T15:46:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741899_1077 (size=6311) 2024-11-23T15:46:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741899_1077 (size=6311) 2024-11-23T15:46:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741899_1077 (size=6311) 2024-11-23T15:46:33,667 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/b/6d03770f60004b2b945be3fee2dc3ae4 to hdfs://localhost:33345/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/b/6d03770f60004b2b945be3fee2dc3ae4 2024-11-23T15:46:33,682 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479, size=6.2 K (6311bytes) 2024-11-23T15:46:33,682 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479 2024-11-23T15:46:33,683 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479 after 1ms 2024-11-23T15:46:33,709 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:33,709 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479 took 27ms 2024-11-23T15:46:33,711 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479 so closing down 2024-11-23T15:46:33,711 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:33,712 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732376793479.temp 2024-11-23T15:46:33,738 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000003-wal.1732376793479.temp 2024-11-23T15:46:33,738 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:33,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741903_1081 (size=5575) 2024-11-23T15:46:33,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741903_1081 (size=5575) 2024-11-23T15:46:33,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741903_1081 (size=5575) 2024-11-23T15:46:33,745 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000003-wal.1732376793479.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:33,747 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000003-wal.1732376793479.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000032 2024-11-23T15:46:33,747 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-23T15:46:33,747 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479, journal: Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479, size=6.2 K (6311bytes) at 1732376793682Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479 so closing down at 1732376793711 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000003-wal.1732376793479.temp at 1732376793738 (+27 ms)3 split writer threads finished at 1732376793738Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000003-wal.1732376793479.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732376793746 (+8 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000003-wal.1732376793479.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000032 at 1732376793747 (+1 ms)Processed 32 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732376793747 2024-11-23T15:46:33,749 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793479 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376793479 2024-11-23T15:46:33,749 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000032 2024-11-23T15:46:33,749 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:33,751 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:33,764 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793751, exclude list is [], retry=0 2024-11-23T15:46:33,767 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:33,767 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:33,767 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:33,792 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793751 2024-11-23T15:46:33,792 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:33,792 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ef896d7e639d709c75b185a128fae935, NAME => 'testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:33,792 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:33,792 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,792 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,794 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,794 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName a 2024-11-23T15:46:33,794 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,799 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/a/034634dbc0ab40c893ddeb452a84eb79 2024-11-23T15:46:33,800 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,800 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,801 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName b 2024-11-23T15:46:33,801 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,802 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,802 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,802 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef896d7e639d709c75b185a128fae935 columnFamilyName c 2024-11-23T15:46:33,803 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,808 DEBUG [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/c/927566bce01546448e0003ffedc2e9f2 2024-11-23T15:46:33,808 INFO [StoreOpener-ef896d7e639d709c75b185a128fae935-1 {}] regionserver.HStore(327): Store=ef896d7e639d709c75b185a128fae935/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,808 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,809 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,811 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,811 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000032 2024-11-23T15:46:33,836 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:33,838 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000032 2024-11-23T15:46:33,838 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ef896d7e639d709c75b185a128fae935 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-23T15:46:33,851 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/b/c6f11d8b031546a998c1cef20a778398 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732376793540/Put/seqid=0 2024-11-23T15:46:33,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741905_1083 (size=5958) 2024-11-23T15:46:33,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741905_1083 (size=5958) 2024-11-23T15:46:33,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741905_1083 (size=5958) 2024-11-23T15:46:33,859 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/b/c6f11d8b031546a998c1cef20a778398 2024-11-23T15:46:33,866 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/.tmp/b/c6f11d8b031546a998c1cef20a778398 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/b/c6f11d8b031546a998c1cef20a778398 2024-11-23T15:46:33,871 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/b/c6f11d8b031546a998c1cef20a778398, entries=10, sequenceid=32, filesize=5.8 K 2024-11-23T15:46:33,872 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ef896d7e639d709c75b185a128fae935 in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-11-23T15:46:33,872 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/0000000000000000032 2024-11-23T15:46:33,874 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,874 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,874 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:33,876 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ef896d7e639d709c75b185a128fae935 2024-11-23T15:46:33,878 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/ef896d7e639d709c75b185a128fae935/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-23T15:46:33,880 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ef896d7e639d709c75b185a128fae935; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59048067, jitterRate=-0.1201152354478836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:33,880 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ef896d7e639d709c75b185a128fae935: Writing region info on filesystem at 1732376793792Initializing all the Stores at 1732376793793 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793793Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793793Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793794 (+1 ms)Obtaining lock to block concurrent updates at 1732376793838 (+44 ms)Preparing flush snapshotting stores in ef896d7e639d709c75b185a128fae935 at 1732376793838Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732376793838Flushing stores of testReplayEditsWrittenViaHRegion,,1732376793407.ef896d7e639d709c75b185a128fae935. at 1732376793838Flushing ef896d7e639d709c75b185a128fae935/b: creating writer at 1732376793838Flushing ef896d7e639d709c75b185a128fae935/b: appending metadata at 1732376793851 (+13 ms)Flushing ef896d7e639d709c75b185a128fae935/b: closing flushed file at 1732376793851Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45ab3be2: reopening flushed file at 1732376793865 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ef896d7e639d709c75b185a128fae935 in 34ms, sequenceid=32, compaction requested=false; wal=null at 1732376793872 (+7 ms)Cleaning up temporary data from old regions at 1732376793874 (+2 ms)Region opened successfully at 1732376793880 (+6 ms) 2024-11-23T15:46:33,905 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:38892 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:54470 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:35516 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:39254 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:55444 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:60510 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1120 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=7691 (was 7695) 2024-11-23T15:46:33,905 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1120 is superior to 1024 2024-11-23T15:46:33,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1120, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=7690 2024-11-23T15:46:33,917 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1120 is superior to 1024 2024-11-23T15:46:33,936 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:33,938 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:33,939 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:33,942 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66569507, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-66569507, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:33,955 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66569507/hregion-66569507.1732376793942, exclude list is [], retry=0 2024-11-23T15:46:33,958 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:33,958 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:33,958 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:33,985 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66569507/hregion-66569507.1732376793942 2024-11-23T15:46:33,985 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:33,985 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1f81d0ee3e30d5648d36fd45c59c6c6c, NAME => 'testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:33,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741907_1085 (size=68) 2024-11-23T15:46:33,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741907_1085 (size=68) 2024-11-23T15:46:33,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741907_1085 (size=68) 2024-11-23T15:46:33,994 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:33,995 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:33,996 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName a 2024-11-23T15:46:33,996 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,997 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,997 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:33,998 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName b 2024-11-23T15:46:33,998 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:33,999 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:33,999 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,000 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName c 2024-11-23T15:46:34,000 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:34,001 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:34,001 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,001 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,003 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,003 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,003 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:34,005 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,007 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:34,007 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f81d0ee3e30d5648d36fd45c59c6c6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70376557, jitterRate=0.04869242012500763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:34,007 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f81d0ee3e30d5648d36fd45c59c6c6c: Writing region info on filesystem at 1732376793994Initializing all the Stores at 1732376793995 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793995Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793995Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376793995Cleaning up temporary data from old regions at 1732376794003 (+8 ms)Region opened successfully at 1732376794007 (+4 ms) 2024-11-23T15:46:34,007 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f81d0ee3e30d5648d36fd45c59c6c6c, disabling compactions & flushes 2024-11-23T15:46:34,007 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:34,008 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:34,008 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. after waiting 0 ms 2024-11-23T15:46:34,008 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:34,008 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:34,008 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f81d0ee3e30d5648d36fd45c59c6c6c: Waiting for close lock at 1732376794007Disabling compacts and flushes for region at 1732376794007Disabling writes for close at 1732376794008 (+1 ms)Writing region close event to WAL at 1732376794008Closed at 1732376794008 2024-11-23T15:46:34,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741906_1084 (size=175) 2024-11-23T15:46:34,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741906_1084 (size=175) 2024-11-23T15:46:34,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741906_1084 (size=175) 2024-11-23T15:46:34,012 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:34,012 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66569507:(num 1732376793942) 2024-11-23T15:46:34,012 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:34,014 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:34,027 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014, exclude list is [], retry=0 2024-11-23T15:46:34,030 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:34,030 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:34,031 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:34,056 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014 2024-11-23T15:46:34,056 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309)] 2024-11-23T15:46:34,131 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f81d0ee3e30d5648d36fd45c59c6c6c, NAME => 'testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:34,134 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,134 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:34,134 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,134 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,136 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,137 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName a 2024-11-23T15:46:34,137 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:34,138 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:34,138 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,139 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName b 2024-11-23T15:46:34,139 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:34,140 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:34,140 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,141 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName c 2024-11-23T15:46:34,141 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:34,141 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:34,142 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,142 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,144 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,145 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,145 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,145 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:34,147 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,148 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f81d0ee3e30d5648d36fd45c59c6c6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70570830, jitterRate=0.051587313413619995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:34,148 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:34,148 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f81d0ee3e30d5648d36fd45c59c6c6c: Running coprocessor pre-open hook at 1732376794134Writing region info on filesystem at 1732376794135 (+1 ms)Initializing all the Stores at 1732376794136 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376794136Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376794136Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376794136Cleaning up temporary data from old regions at 1732376794145 (+9 ms)Running coprocessor post-open hooks at 1732376794148 (+3 ms)Region opened successfully at 1732376794148 2024-11-23T15:46:34,161 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-23T15:46:34,162 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:34,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:46:35,163 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:36,165 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:37,166 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:38,168 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:39,169 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:39,639 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:46:40,171 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:40,665 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:46:41,172 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:41,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-23T15:46:41,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:41,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T15:46:41,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:41,818 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-23T15:46:41,818 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:41,818 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-23T15:46:41,818 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:41,819 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:46:41,819 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T15:46:42,173 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:43,174 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 1f81d0ee3e30d5648d36fd45c59c6c6c/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:43,176 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f81d0ee3e30d5648d36fd45c59c6c6c: 2024-11-23T15:46:43,176 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:43,193 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f81d0ee3e30d5648d36fd45c59c6c6c: 2024-11-23T15:46:43,193 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-23T15:46:43,193 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f81d0ee3e30d5648d36fd45c59c6c6c, disabling compactions & flushes 2024-11-23T15:46:43,193 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:43,193 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:43,193 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. after waiting 0 ms 2024-11-23T15:46:43,193 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:43,193 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:43,194 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. 2024-11-23T15:46:43,194 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f81d0ee3e30d5648d36fd45c59c6c6c: Waiting for close lock at 1732376803193Running coprocessor pre-close hooks at 1732376803193Disabling compacts and flushes for region at 1732376803193Disabling writes for close at 1732376803193Writing region close event to WAL at 1732376803193Running coprocessor post-close hooks at 1732376803194 (+1 ms)Closed at 1732376803194 2024-11-23T15:46:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741908_1086 (size=4236) 2024-11-23T15:46:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741908_1086 (size=4236) 2024-11-23T15:46:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741908_1086 (size=4236) 2024-11-23T15:46:43,210 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014, size=4.1 K (4236bytes) 2024-11-23T15:46:43,210 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014 2024-11-23T15:46:43,210 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014 after 0ms 2024-11-23T15:46:43,236 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:43,236 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014 took 27ms 2024-11-23T15:46:43,238 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014 so closing down 2024-11-23T15:46:43,239 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:43,240 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732376794014.temp 2024-11-23T15:46:43,272 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000004-wal.1732376794014.temp 2024-11-23T15:46:43,273 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741909_1087 (size=3245) 2024-11-23T15:46:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741909_1087 (size=3245) 2024-11-23T15:46:43,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741909_1087 (size=3245) 2024-11-23T15:46:43,279 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000004-wal.1732376794014.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:43,281 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000004-wal.1732376794014.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000026 2024-11-23T15:46:43,281 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 44 ms; skipped=3; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-23T15:46:43,281 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014, journal: Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014, size=4.1 K (4236bytes) at 1732376803210Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014 so closing down at 1732376803239 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000004-wal.1732376794014.temp at 1732376803272 (+33 ms)3 split writer threads finished at 1732376803273 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000004-wal.1732376794014.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732376803279 (+6 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000004-wal.1732376794014.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000026 at 1732376803281 (+2 ms)Processed 23 edits across 1 Regions in 44 ms; skipped=3; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732376803281 2024-11-23T15:46:43,283 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376794014 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376794014 2024-11-23T15:46:43,283 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000026 2024-11-23T15:46:43,284 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:43,285 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:43,303 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376803286, exclude list is [], retry=0 2024-11-23T15:46:43,306 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:43,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:43,307 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:43,334 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376803286 2024-11-23T15:46:43,334 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:43,335 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f81d0ee3e30d5648d36fd45c59c6c6c, NAME => 'testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:43,335 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,335 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:43,335 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,335 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,337 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,338 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName a 2024-11-23T15:46:43,338 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:43,338 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:43,338 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,339 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName b 2024-11-23T15:46:43,339 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:43,340 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:43,340 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,341 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f81d0ee3e30d5648d36fd45c59c6c6c columnFamilyName c 2024-11-23T15:46:43,341 DEBUG [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:43,341 INFO [StoreOpener-1f81d0ee3e30d5648d36fd45c59c6c6c-1 {}] regionserver.HStore(327): Store=1f81d0ee3e30d5648d36fd45c59c6c6c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:43,341 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,344 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,345 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000026 2024-11-23T15:46:43,379 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:43,381 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000026 2024-11-23T15:46:43,382 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-23T15:46:43,396 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/a/5214d54bcce341af86e85d93bced495d is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732376803183/Put/seqid=0 2024-11-23T15:46:43,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741911_1089 (size=5523) 2024-11-23T15:46:43,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741911_1089 (size=5523) 2024-11-23T15:46:43,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741911_1089 (size=5523) 2024-11-23T15:46:43,404 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/a/5214d54bcce341af86e85d93bced495d 2024-11-23T15:46:43,426 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/b/d3866f86bda349a79f8bbb2dc0be4a04 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732376803177/Put/seqid=0 2024-11-23T15:46:43,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741912_1090 (size=5524) 2024-11-23T15:46:43,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741912_1090 (size=5524) 2024-11-23T15:46:43,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741912_1090 (size=5524) 2024-11-23T15:46:43,433 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/b/d3866f86bda349a79f8bbb2dc0be4a04 2024-11-23T15:46:43,459 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/c/e4d6c6f77bd54783b9490479cebda526 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732376803180/Put/seqid=0 2024-11-23T15:46:43,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741913_1091 (size=5457) 2024-11-23T15:46:43,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741913_1091 (size=5457) 2024-11-23T15:46:43,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741913_1091 (size=5457) 2024-11-23T15:46:43,466 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/c/e4d6c6f77bd54783b9490479cebda526 2024-11-23T15:46:43,472 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/a/5214d54bcce341af86e85d93bced495d as hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/a/5214d54bcce341af86e85d93bced495d 2024-11-23T15:46:43,477 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/a/5214d54bcce341af86e85d93bced495d, entries=7, sequenceid=26, filesize=5.4 K 2024-11-23T15:46:43,478 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/b/d3866f86bda349a79f8bbb2dc0be4a04 as hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/b/d3866f86bda349a79f8bbb2dc0be4a04 2024-11-23T15:46:43,483 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/b/d3866f86bda349a79f8bbb2dc0be4a04, entries=7, sequenceid=26, filesize=5.4 K 2024-11-23T15:46:43,484 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/.tmp/c/e4d6c6f77bd54783b9490479cebda526 as hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/c/e4d6c6f77bd54783b9490479cebda526 2024-11-23T15:46:43,489 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/c/e4d6c6f77bd54783b9490479cebda526, entries=6, sequenceid=26, filesize=5.3 K 2024-11-23T15:46:43,489 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 1f81d0ee3e30d5648d36fd45c59c6c6c in 108ms, sequenceid=26, compaction requested=false; wal=null 2024-11-23T15:46:43,490 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/0000000000000000026 2024-11-23T15:46:43,491 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,491 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,492 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:43,493 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,495 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsAfterAbortingFlush/1f81d0ee3e30d5648d36fd45c59c6c6c/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-23T15:46:43,496 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f81d0ee3e30d5648d36fd45c59c6c6c; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74878720, jitterRate=0.11577987670898438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:43,496 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f81d0ee3e30d5648d36fd45c59c6c6c 2024-11-23T15:46:43,497 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f81d0ee3e30d5648d36fd45c59c6c6c: Running coprocessor pre-open hook at 1732376803335Writing region info on filesystem at 1732376803335Initializing all the Stores at 1732376803336 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376803336Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376803337 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376803337Obtaining lock to block concurrent updates at 1732376803382 (+45 ms)Preparing flush snapshotting stores in 1f81d0ee3e30d5648d36fd45c59c6c6c at 1732376803382Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732376803382Flushing stores of testReplayEditsAfterAbortingFlush,,1732376793937.1f81d0ee3e30d5648d36fd45c59c6c6c. at 1732376803382Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/a: creating writer at 1732376803382Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/a: appending metadata at 1732376803396 (+14 ms)Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/a: closing flushed file at 1732376803396Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/b: creating writer at 1732376803410 (+14 ms)Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/b: appending metadata at 1732376803426 (+16 ms)Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/b: closing flushed file at 1732376803426Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/c: creating writer at 1732376803439 (+13 ms)Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/c: appending metadata at 1732376803459 (+20 ms)Flushing 1f81d0ee3e30d5648d36fd45c59c6c6c/c: closing flushed file at 1732376803459Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e4691e0: reopening flushed file at 1732376803471 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@340c8f3a: reopening flushed file at 1732376803477 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3daefda4: reopening flushed file at 1732376803483 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 1f81d0ee3e30d5648d36fd45c59c6c6c in 108ms, sequenceid=26, compaction requested=false; wal=null at 1732376803489 (+6 ms)Cleaning up temporary data from old regions at 1732376803491 (+2 ms)Running coprocessor post-open hooks at 1732376803496 (+5 ms)Region opened successfully at 1732376803497 (+1 ms) 2024-11-23T15:46:43,514 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417 (was 419), OpenFileDescriptor=1182 (was 1120) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=7695 (was 7690) - AvailableMemoryMB LEAK? - 2024-11-23T15:46:43,514 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-23T15:46:43,525 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=7694 2024-11-23T15:46:43,525 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-23T15:46:43,537 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:43,538 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:43,539 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:43,541 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45296743, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-45296743, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:43,552 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45296743/hregion-45296743.1732376803541, exclude list is [], retry=0 2024-11-23T15:46:43,555 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:43,555 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:43,555 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:43,581 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45296743/hregion-45296743.1732376803541 2024-11-23T15:46:43,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:43,591 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9d7d2856d69b09e89d9f74db75ee926f, NAME => 'testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:43,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741915_1093 (size=61) 2024-11-23T15:46:43,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741915_1093 (size=61) 2024-11-23T15:46:43,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741915_1093 (size=61) 2024-11-23T15:46:43,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:43,602 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,603 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d7d2856d69b09e89d9f74db75ee926f columnFamilyName a 2024-11-23T15:46:43,603 DEBUG [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:43,603 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(327): Store=9d7d2856d69b09e89d9f74db75ee926f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:43,604 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,604 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,604 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,605 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,605 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,606 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,609 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:43,609 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d7d2856d69b09e89d9f74db75ee926f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74719286, jitterRate=0.11340412497520447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:43,609 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d7d2856d69b09e89d9f74db75ee926f: Writing region info on filesystem at 1732376803601Initializing all the Stores at 1732376803601Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376803601Cleaning up temporary data from old regions at 1732376803605 (+4 ms)Region opened successfully at 1732376803609 (+4 ms) 2024-11-23T15:46:43,609 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9d7d2856d69b09e89d9f74db75ee926f, disabling compactions & flushes 2024-11-23T15:46:43,610 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,610 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,610 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. after waiting 0 ms 2024-11-23T15:46:43,610 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,610 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,610 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9d7d2856d69b09e89d9f74db75ee926f: Waiting for close lock at 1732376803609Disabling compacts and flushes for region at 1732376803609Disabling writes for close at 1732376803610 (+1 ms)Writing region close event to WAL at 1732376803610Closed at 1732376803610 2024-11-23T15:46:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741914_1092 (size=175) 2024-11-23T15:46:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741914_1092 (size=175) 2024-11-23T15:46:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741914_1092 (size=175) 2024-11-23T15:46:43,615 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:43,615 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45296743:(num 1732376803541) 2024-11-23T15:46:43,615 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:43,617 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:43,629 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617, exclude list is [], retry=0 2024-11-23T15:46:43,632 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:43,632 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:43,632 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:43,657 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617 2024-11-23T15:46:43,657 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:43,657 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d7d2856d69b09e89d9f74db75ee926f, NAME => 'testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:43,657 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:43,658 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,658 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,659 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,660 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d7d2856d69b09e89d9f74db75ee926f columnFamilyName a 2024-11-23T15:46:43,660 DEBUG [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:43,660 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(327): Store=9d7d2856d69b09e89d9f74db75ee926f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:43,660 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,661 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,662 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,662 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,662 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,664 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:43,665 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d7d2856d69b09e89d9f74db75ee926f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63725186, jitterRate=-0.05042073130607605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:43,665 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d7d2856d69b09e89d9f74db75ee926f: Writing region info on filesystem at 1732376803658Initializing all the Stores at 1732376803659 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376803659Cleaning up temporary data from old regions at 1732376803662 (+3 ms)Region opened successfully at 1732376803665 (+3 ms) 2024-11-23T15:46:43,673 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9d7d2856d69b09e89d9f74db75ee926f, disabling compactions & flushes 2024-11-23T15:46:43,673 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,673 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,673 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. after waiting 0 ms 2024-11-23T15:46:43,673 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,673 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,674 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. 2024-11-23T15:46:43,674 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9d7d2856d69b09e89d9f74db75ee926f: Waiting for close lock at 1732376803673Disabling compacts and flushes for region at 1732376803673Disabling writes for close at 1732376803673Writing region close event to WAL at 1732376803673Closed at 1732376803674 (+1 ms) 2024-11-23T15:46:43,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741916_1094 (size=1795) 2024-11-23T15:46:43,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741916_1094 (size=1795) 2024-11-23T15:46:43,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741916_1094 (size=1795) 2024-11-23T15:46:43,691 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617, size=1.8 K (1795bytes) 2024-11-23T15:46:43,691 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617 2024-11-23T15:46:43,692 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617 after 1ms 2024-11-23T15:46:43,715 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:43,715 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617 took 24ms 2024-11-23T15:46:43,717 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617 so closing down 2024-11-23T15:46:43,717 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:43,719 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732376803617.temp 2024-11-23T15:46:43,743 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000003-wal.1732376803617.temp 2024-11-23T15:46:43,743 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741917_1095 (size=1795) 2024-11-23T15:46:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741917_1095 (size=1795) 2024-11-23T15:46:43,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741917_1095 (size=1795) 2024-11-23T15:46:43,751 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000003-wal.1732376803617.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:43,752 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000003-wal.1732376803617.temp to hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000012 2024-11-23T15:46:43,753 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-23T15:46:43,753 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617, journal: Splitting hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617, size=1.8 K (1795bytes) at 1732376803691Finishing writing output for hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617 so closing down at 1732376803717 (+26 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000003-wal.1732376803617.temp at 1732376803743 (+26 ms)3 split writer threads finished at 1732376803743Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000003-wal.1732376803617.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732376803751 (+8 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000003-wal.1732376803617.temp to hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000012 at 1732376803752 (+1 ms)Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732376803753 (+1 ms) 2024-11-23T15:46:43,754 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376803617 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376803617 2024-11-23T15:46:43,755 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000012 2024-11-23T15:46:43,781 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:44,097 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:44,100 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:44,111 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376804100, exclude list is [], retry=0 2024-11-23T15:46:44,113 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:44,114 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:44,114 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:44,137 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376804100 2024-11-23T15:46:44,137 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:44,138 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d7d2856d69b09e89d9f74db75ee926f, NAME => 'testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:44,138 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:44,138 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,138 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,141 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,143 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d7d2856d69b09e89d9f74db75ee926f columnFamilyName a 2024-11-23T15:46:44,143 DEBUG [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,144 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(327): Store=9d7d2856d69b09e89d9f74db75ee926f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,144 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,145 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,146 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,147 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000012 2024-11-23T15:46:44,172 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:44,173 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000012 2024-11-23T15:46:44,174 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d7d2856d69b09e89d9f74db75ee926f 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-23T15:46:44,189 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/.tmp/a/5a4840d0932f42f6aed2d392f7fe0b2a is 79, key is testDatalossWhenInputError/a:x0/1732376803665/Put/seqid=0 2024-11-23T15:46:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741919_1097 (size=5808) 2024-11-23T15:46:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741919_1097 (size=5808) 2024-11-23T15:46:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741919_1097 (size=5808) 2024-11-23T15:46:44,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/.tmp/a/5a4840d0932f42f6aed2d392f7fe0b2a 2024-11-23T15:46:44,206 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/.tmp/a/5a4840d0932f42f6aed2d392f7fe0b2a as hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/a/5a4840d0932f42f6aed2d392f7fe0b2a 2024-11-23T15:46:44,216 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/a/5a4840d0932f42f6aed2d392f7fe0b2a, entries=10, sequenceid=12, filesize=5.7 K 2024-11-23T15:46:44,216 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9d7d2856d69b09e89d9f74db75ee926f in 43ms, sequenceid=12, compaction requested=false; wal=null 2024-11-23T15:46:44,217 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/0000000000000000012 2024-11-23T15:46:44,217 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,218 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,220 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,222 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-23T15:46:44,223 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d7d2856d69b09e89d9f74db75ee926f; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63851258, jitterRate=-0.04854211211204529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:44,224 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d7d2856d69b09e89d9f74db75ee926f: Writing region info on filesystem at 1732376804138Initializing all the Stores at 1732376804140 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804140Obtaining lock to block concurrent updates at 1732376804174 (+34 ms)Preparing flush snapshotting stores in 9d7d2856d69b09e89d9f74db75ee926f at 1732376804174Finished memstore snapshotting testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732376804174Flushing stores of testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f. at 1732376804174Flushing 9d7d2856d69b09e89d9f74db75ee926f/a: creating writer at 1732376804174Flushing 9d7d2856d69b09e89d9f74db75ee926f/a: appending metadata at 1732376804189 (+15 ms)Flushing 9d7d2856d69b09e89d9f74db75ee926f/a: closing flushed file at 1732376804189Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@722f487f: reopening flushed file at 1732376804204 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9d7d2856d69b09e89d9f74db75ee926f in 43ms, sequenceid=12, compaction requested=false; wal=null at 1732376804216 (+12 ms)Cleaning up temporary data from old regions at 1732376804218 (+2 ms)Region opened successfully at 1732376804223 (+5 ms) 2024-11-23T15:46:44,226 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d7d2856d69b09e89d9f74db75ee926f, NAME => 'testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:44,226 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732376803537.9d7d2856d69b09e89d9f74db75ee926f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:44,227 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,227 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,228 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,229 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d7d2856d69b09e89d9f74db75ee926f columnFamilyName a 2024-11-23T15:46:44,230 DEBUG [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,235 DEBUG [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/a/5a4840d0932f42f6aed2d392f7fe0b2a 2024-11-23T15:46:44,236 INFO [StoreOpener-9d7d2856d69b09e89d9f74db75ee926f-1 {}] regionserver.HStore(327): Store=9d7d2856d69b09e89d9f74db75ee926f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,236 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,236 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,238 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,238 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,240 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d7d2856d69b09e89d9f74db75ee926f 2024-11-23T15:46:44,242 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testDatalossWhenInputError/9d7d2856d69b09e89d9f74db75ee926f/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-23T15:46:44,243 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d7d2856d69b09e89d9f74db75ee926f; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62950283, jitterRate=-0.06196768581867218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T15:46:44,243 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d7d2856d69b09e89d9f74db75ee926f: Writing region info on filesystem at 1732376804227Initializing all the Stores at 1732376804228 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804228Cleaning up temporary data from old regions at 1732376804238 (+10 ms)Region opened successfully at 1732376804243 (+5 ms) 2024-11-23T15:46:44,259 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=433 (was 417) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:56486 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1981799117-172.17.0.2-1732376767483:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:44360 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:56404 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:60536 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:44312 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:60628 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 251), ProcessCount=11 (was 11), AvailableMemoryMB=7678 (was 7694) 2024-11-23T15:46:44,259 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-23T15:46:44,271 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=433, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=7678 2024-11-23T15:46:44,272 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-23T15:46:44,284 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:44,286 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:44,286 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:44,288 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-47624436, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-47624436, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:44,299 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-47624436/hregion-47624436.1732376804289, exclude list is [], retry=0 2024-11-23T15:46:44,302 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:44,302 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:44,302 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:44,328 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-47624436/hregion-47624436.1732376804289 2024-11-23T15:46:44,328 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309)] 2024-11-23T15:46:44,328 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3ea5840a0edea350a2748df8df64d53f, NAME => 'testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:44,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741921_1099 (size=63) 2024-11-23T15:46:44,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741921_1099 (size=63) 2024-11-23T15:46:44,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741921_1099 (size=63) 2024-11-23T15:46:44,337 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:44,338 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,339 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName a 2024-11-23T15:46:44,339 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,340 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,340 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,341 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName b 2024-11-23T15:46:44,341 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,341 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,341 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,342 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName c 2024-11-23T15:46:44,342 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,343 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,343 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,344 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,344 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,345 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,345 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,346 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:44,347 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,349 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:44,349 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ea5840a0edea350a2748df8df64d53f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73489857, jitterRate=0.09508420526981354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:44,350 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ea5840a0edea350a2748df8df64d53f: Writing region info on filesystem at 1732376804337Initializing all the Stores at 1732376804338 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804338Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804338Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804338Cleaning up temporary data from old regions at 1732376804345 (+7 ms)Region opened successfully at 1732376804350 (+5 ms) 2024-11-23T15:46:44,350 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3ea5840a0edea350a2748df8df64d53f, disabling compactions & flushes 2024-11-23T15:46:44,350 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:44,350 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:44,350 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. after waiting 0 ms 2024-11-23T15:46:44,350 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:44,350 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:44,351 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3ea5840a0edea350a2748df8df64d53f: Waiting for close lock at 1732376804350Disabling compacts and flushes for region at 1732376804350Disabling writes for close at 1732376804350Writing region close event to WAL at 1732376804350Closed at 1732376804350 2024-11-23T15:46:44,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741920_1098 (size=175) 2024-11-23T15:46:44,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741920_1098 (size=175) 2024-11-23T15:46:44,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741920_1098 (size=175) 2024-11-23T15:46:44,355 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:44,355 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-47624436:(num 1732376804289) 2024-11-23T15:46:44,355 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:44,357 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:44,371 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357, exclude list is [], retry=0 2024-11-23T15:46:44,373 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:44,374 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:44,374 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:44,400 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 2024-11-23T15:46:44,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:44,400 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ea5840a0edea350a2748df8df64d53f, NAME => 'testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:44,400 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:44,400 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,401 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,402 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,402 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName a 2024-11-23T15:46:44,403 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,403 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,403 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,404 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName b 2024-11-23T15:46:44,404 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,404 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,404 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,405 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName c 2024-11-23T15:46:44,405 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:44,406 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:44,406 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,407 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,409 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,409 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,409 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:44,411 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:44,411 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ea5840a0edea350a2748df8df64d53f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61949427, jitterRate=-0.07688160240650177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:44,412 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ea5840a0edea350a2748df8df64d53f: Writing region info on filesystem at 1732376804401Initializing all the Stores at 1732376804401Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804401Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804402 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376804402Cleaning up temporary data from old regions at 1732376804409 (+7 ms)Region opened successfully at 1732376804412 (+3 ms) 2024-11-23T15:46:44,417 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732376804416/Put/seqid=0 2024-11-23T15:46:44,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741923_1101 (size=4875) 2024-11-23T15:46:44,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741923_1101 (size=4875) 2024-11-23T15:46:44,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741923_1101 (size=4875) 2024-11-23T15:46:44,427 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732376804426/Put/seqid=0 2024-11-23T15:46:44,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741924_1102 (size=4875) 2024-11-23T15:46:44,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741924_1102 (size=4875) 2024-11-23T15:46:44,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741924_1102 (size=4875) 2024-11-23T15:46:44,436 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732376804435/Put/seqid=0 2024-11-23T15:46:44,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741925_1103 (size=4875) 2024-11-23T15:46:44,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741925_1103 (size=4875) 2024-11-23T15:46:44,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741925_1103 (size=4875) 2024-11-23T15:46:44,442 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 3ea5840a0edea350a2748df8df64d53f/a 2024-11-23T15:46:44,445 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-23T15:46:44,445 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T15:46:44,445 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 3ea5840a0edea350a2748df8df64d53f/a 2024-11-23T15:46:44,449 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-23T15:46:44,449 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T15:46:44,449 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 3ea5840a0edea350a2748df8df64d53f/a 2024-11-23T15:46:44,452 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-23T15:46:44,452 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-23T15:46:44,452 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ea5840a0edea350a2748df8df64d53f 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-23T15:46:44,466 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/.tmp/a/ffdb6a52f0964d16b4df888c9622a3db is 55, key is testCompactedBulkLoadedFiles/a:a/1732376804413/Put/seqid=0 2024-11-23T15:46:44,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741926_1104 (size=5107) 2024-11-23T15:46:44,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741926_1104 (size=5107) 2024-11-23T15:46:44,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741926_1104 (size=5107) 2024-11-23T15:46:44,473 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/.tmp/a/ffdb6a52f0964d16b4df888c9622a3db 2024-11-23T15:46:44,479 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/.tmp/a/ffdb6a52f0964d16b4df888c9622a3db as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db 2024-11-23T15:46:44,484 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db, entries=1, sequenceid=4, filesize=5.0 K 2024-11-23T15:46:44,485 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 3ea5840a0edea350a2748df8df64d53f in 33ms, sequenceid=4, compaction requested=false 2024-11-23T15:46:44,485 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3ea5840a0edea350a2748df8df64d53f: 2024-11-23T15:46:44,486 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_ 2024-11-23T15:46:44,487 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_ 2024-11-23T15:46:44,488 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_ 2024-11-23T15:46:44,488 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile0 into 3ea5840a0edea350a2748df8df64d53f/a as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_ - updating store file list. 2024-11-23T15:46:44,493 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8b24a5329aea416ebda8021b57916691_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:44,493 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_ into 3ea5840a0edea350a2748df8df64d53f/a 2024-11-23T15:46:44,493 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile0 into 3ea5840a0edea350a2748df8df64d53f/a (new location: hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_) 2024-11-23T15:46:44,494 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile1 into 3ea5840a0edea350a2748df8df64d53f/a as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_ - updating store file list. 2024-11-23T15:46:44,498 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 860646fd80b9408685b15c277e1e37c0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:44,498 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_ into 3ea5840a0edea350a2748df8df64d53f/a 2024-11-23T15:46:44,498 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile1 into 3ea5840a0edea350a2748df8df64d53f/a (new location: hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_) 2024-11-23T15:46:44,499 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile2 into 3ea5840a0edea350a2748df8df64d53f/a as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_ - updating store file list. 2024-11-23T15:46:44,503 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d032e1f4c193400d9b743aabf34189b3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:44,503 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_ into 3ea5840a0edea350a2748df8df64d53f/a 2024-11-23T15:46:44,503 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33345/hbase/testCompactedBulkLoadedFiles/hfile2 into 3ea5840a0edea350a2748df8df64d53f/a (new location: hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_) 2024-11-23T15:46:44,510 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:46:44,510 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3ea5840a0edea350a2748df8df64d53f/a is initiating major compaction (all files) 2024-11-23T15:46:44,510 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3ea5840a0edea350a2748df8df64d53f/a in testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:44,510 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_] into tmpdir=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/.tmp, totalSize=19.3 K 2024-11-23T15:46:44,511 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ffdb6a52f0964d16b4df888c9622a3db, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732376804413 2024-11-23T15:46:44,511 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8b24a5329aea416ebda8021b57916691_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T15:46:44,512 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 860646fd80b9408685b15c277e1e37c0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T15:46:44,512 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d032e1f4c193400d9b743aabf34189b3_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-23T15:46:44,524 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/.tmp/a/a528546fd187408db29f2dddd0a7653a is 55, key is testCompactedBulkLoadedFiles/a:a/1732376804413/Put/seqid=0 2024-11-23T15:46:44,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741927_1105 (size=6154) 2024-11-23T15:46:44,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741927_1105 (size=6154) 2024-11-23T15:46:44,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741927_1105 (size=6154) 2024-11-23T15:46:44,541 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/.tmp/a/a528546fd187408db29f2dddd0a7653a as hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/a528546fd187408db29f2dddd0a7653a 2024-11-23T15:46:44,547 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 3ea5840a0edea350a2748df8df64d53f/a of 3ea5840a0edea350a2748df8df64d53f into a528546fd187408db29f2dddd0a7653a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:46:44,547 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3ea5840a0edea350a2748df8df64d53f: 2024-11-23T15:46:44,547 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-23T15:46:44,547 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-23T15:46:44,578 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357, size=0 (0bytes) 2024-11-23T15:46:44,578 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 might be still open, length is 0 2024-11-23T15:46:44,578 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 2024-11-23T15:46:44,579 WARN [IPC Server handler 1 on default port 33345 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-23T15:46:44,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 after 1ms 2024-11-23T15:46:46,582 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:60666 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:35471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60666 dst: /127.0.0.1:35471 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35471 remote=/127.0.0.1:60666]. Total timeout mills is 60000, 57964 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:46,583 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:56524 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:44917:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56524 dst: /127.0.0.1:44917 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:46,583 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:44398 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37735:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44398 dst: /127.0.0.1:37735 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:46,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741922_1106 (size=1794) 2024-11-23T15:46:46,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741922_1106 (size=1794) 2024-11-23T15:46:48,581 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 after 4003ms 2024-11-23T15:46:48,611 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:48,611 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 took 4033ms 2024-11-23T15:46:48,613 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357; continuing. 2024-11-23T15:46:48,613 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 so closing down 2024-11-23T15:46:48,614 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:48,615 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732376804357.temp 2024-11-23T15:46:48,640 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000003-wal.1732376804357.temp 2024-11-23T15:46:48,640 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:48,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741928_1107 (size=800) 2024-11-23T15:46:48,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741928_1107 (size=800) 2024-11-23T15:46:48,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741928_1107 (size=800) 2024-11-23T15:46:48,647 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000003-wal.1732376804357.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:48,649 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000003-wal.1732376804357.temp to hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000008 2024-11-23T15:46:48,649 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T15:46:48,649 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357, journal: Splitting hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357, size=0 (0bytes) at 1732376804578Finishing writing output for hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 so closing down at 1732376808613 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000003-wal.1732376804357.temp at 1732376808640 (+27 ms)3 split writer threads finished at 1732376808641 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000003-wal.1732376804357.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732376808647 (+6 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000003-wal.1732376804357.temp to hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000008 at 1732376808649 (+2 ms)Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357, size=0, length=0, corrupted=false, cancelled=false at 1732376808649 2024-11-23T15:46:48,651 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376804357 2024-11-23T15:46:48,651 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000008 2024-11-23T15:46:48,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:48,653 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:48,664 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376808653, exclude list is [], retry=0 2024-11-23T15:46:48,666 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:48,667 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:48,667 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:48,699 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:46:48,700 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376808653 2024-11-23T15:46:48,700 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:48,700 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ea5840a0edea350a2748df8df64d53f, NAME => 'testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:48,701 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:48,701 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,701 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,702 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,703 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName a 2024-11-23T15:46:48,703 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,711 DEBUG [StoreFileOpener-3ea5840a0edea350a2748df8df64d53f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 860646fd80b9408685b15c277e1e37c0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:48,711 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_ 2024-11-23T15:46:48,719 DEBUG [StoreFileOpener-3ea5840a0edea350a2748df8df64d53f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8b24a5329aea416ebda8021b57916691_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:48,720 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_ 2024-11-23T15:46:48,725 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/a528546fd187408db29f2dddd0a7653a 2024-11-23T15:46:48,730 DEBUG [StoreFileOpener-3ea5840a0edea350a2748df8df64d53f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d032e1f4c193400d9b743aabf34189b3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-23T15:46:48,730 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_ 2024-11-23T15:46:48,735 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db 2024-11-23T15:46:48,735 WARN [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5f52253 2024-11-23T15:46:48,735 WARN [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5f52253 2024-11-23T15:46:48,735 WARN [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5f52253 2024-11-23T15:46:48,735 WARN [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5f52253 2024-11-23T15:46:48,735 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db] to archive 2024-11-23T15:46:48,736 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:46:48,738 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_ to hdfs://localhost:33345/hbase/archive/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/860646fd80b9408685b15c277e1e37c0_SeqId_4_ 2024-11-23T15:46:48,740 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_ to hdfs://localhost:33345/hbase/archive/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/8b24a5329aea416ebda8021b57916691_SeqId_4_ 2024-11-23T15:46:48,741 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_ to hdfs://localhost:33345/hbase/archive/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/d032e1f4c193400d9b743aabf34189b3_SeqId_4_ 2024-11-23T15:46:48,743 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db to hdfs://localhost:33345/hbase/archive/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/a/ffdb6a52f0964d16b4df888c9622a3db 2024-11-23T15:46:48,743 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,743 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,744 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName b 2024-11-23T15:46:48,744 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,745 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,745 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,746 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea5840a0edea350a2748df8df64d53f columnFamilyName c 2024-11-23T15:46:48,746 DEBUG [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,746 INFO [StoreOpener-3ea5840a0edea350a2748df8df64d53f-1 {}] regionserver.HStore(327): Store=3ea5840a0edea350a2748df8df64d53f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,746 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,747 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,749 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,749 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000008 2024-11-23T15:46:48,776 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:48,778 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 3ea5840a0edea350a2748df8df64d53f : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "3ea5840a0edea350a2748df8df64d53f" family_name: "a" compaction_input: "ffdb6a52f0964d16b4df888c9622a3db" compaction_input: "8b24a5329aea416ebda8021b57916691_SeqId_4_" compaction_input: "860646fd80b9408685b15c277e1e37c0_SeqId_4_" compaction_input: "d032e1f4c193400d9b743aabf34189b3_SeqId_4_" compaction_output: "a528546fd187408db29f2dddd0a7653a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-23T15:46:48,779 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-23T15:46:48,779 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000008 2024-11-23T15:46:48,780 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/0000000000000000008 2024-11-23T15:46:48,781 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,781 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,781 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:48,783 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ea5840a0edea350a2748df8df64d53f 2024-11-23T15:46:48,785 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testCompactedBulkLoadedFiles/3ea5840a0edea350a2748df8df64d53f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-23T15:46:48,786 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ea5840a0edea350a2748df8df64d53f; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65128763, jitterRate=-0.029505804181098938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:48,787 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ea5840a0edea350a2748df8df64d53f: Writing region info on filesystem at 1732376808701Initializing all the Stores at 1732376808702 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808702Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808702Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808702Cleaning up temporary data from old regions at 1732376808781 (+79 ms)Region opened successfully at 1732376808786 (+5 ms) 2024-11-23T15:46:48,790 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3ea5840a0edea350a2748df8df64d53f, disabling compactions & flushes 2024-11-23T15:46:48,790 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:48,790 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:48,790 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. after waiting 0 ms 2024-11-23T15:46:48,790 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:48,791 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732376804285.3ea5840a0edea350a2748df8df64d53f. 2024-11-23T15:46:48,791 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3ea5840a0edea350a2748df8df64d53f: Waiting for close lock at 1732376808790Disabling compacts and flushes for region at 1732376808790Disabling writes for close at 1732376808790Writing region close event to WAL at 1732376808791 (+1 ms)Closed at 1732376808791 2024-11-23T15:46:48,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741929_1108 (size=175) 2024-11-23T15:46:48,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741929_1108 (size=175) 2024-11-23T15:46:48,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741929_1108 (size=175) 2024-11-23T15:46:48,797 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:48,797 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732376808653) 2024-11-23T15:46:48,815 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 433) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1729117561_22 at /127.0.0.1:60730 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33345 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:33345 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1729117561_22 at /127.0.0.1:44460 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1729117561_22 at /127.0.0.1:56626 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 251), ProcessCount=11 (was 11), AvailableMemoryMB=7659 (was 7678) 2024-11-23T15:46:48,815 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-23T15:46:48,831 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=7657 2024-11-23T15:46:48,831 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-23T15:46:48,851 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:48,853 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:46:48,853 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:46:48,856 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-28011367, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/hregion-28011367, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:48,868 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-28011367/hregion-28011367.1732376808857, exclude list is [], retry=0 2024-11-23T15:46:48,871 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:48,871 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:48,871 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:48,896 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-28011367/hregion-28011367.1732376808857 2024-11-23T15:46:48,896 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:48,896 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0267affe6d2f53bbc9948e282743e54c, NAME => 'testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33345/hbase 2024-11-23T15:46:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741931_1110 (size=67) 2024-11-23T15:46:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741931_1110 (size=67) 2024-11-23T15:46:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741931_1110 (size=67) 2024-11-23T15:46:48,906 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:48,907 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,908 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName a 2024-11-23T15:46:48,908 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,908 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,908 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,909 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName b 2024-11-23T15:46:48,910 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,910 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,910 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,911 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName c 2024-11-23T15:46:48,911 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,911 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,911 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,913 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,913 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,913 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:48,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,916 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:46:48,917 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0267affe6d2f53bbc9948e282743e54c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75360897, jitterRate=0.12296487390995026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:48,917 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0267affe6d2f53bbc9948e282743e54c: Writing region info on filesystem at 1732376808906Initializing all the Stores at 1732376808906Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808906Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808907 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808907Cleaning up temporary data from old regions at 1732376808913 (+6 ms)Region opened successfully at 1732376808917 (+4 ms) 2024-11-23T15:46:48,917 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0267affe6d2f53bbc9948e282743e54c, disabling compactions & flushes 2024-11-23T15:46:48,917 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:48,917 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:48,917 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. after waiting 0 ms 2024-11-23T15:46:48,917 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:48,917 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:48,918 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0267affe6d2f53bbc9948e282743e54c: Waiting for close lock at 1732376808917Disabling compacts and flushes for region at 1732376808917Disabling writes for close at 1732376808917Writing region close event to WAL at 1732376808917Closed at 1732376808917 2024-11-23T15:46:48,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741930_1109 (size=175) 2024-11-23T15:46:48,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741930_1109 (size=175) 2024-11-23T15:46:48,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741930_1109 (size=175) 2024-11-23T15:46:48,922 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:48,922 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-28011367:(num 1732376808857) 2024-11-23T15:46:48,922 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:48,924 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:48,940 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924, exclude list is [], retry=0 2024-11-23T15:46:48,943 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:48,944 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:48,944 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:48,970 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924 2024-11-23T15:46:48,971 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:48,971 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0267affe6d2f53bbc9948e282743e54c, NAME => 'testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:48,971 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:48,971 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,971 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,973 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,973 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName a 2024-11-23T15:46:48,974 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,974 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,974 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,975 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName b 2024-11-23T15:46:48,975 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,975 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,975 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,976 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName c 2024-11-23T15:46:48,976 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:48,976 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:48,976 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,977 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,979 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,979 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,979 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:48,980 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:48,981 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0267affe6d2f53bbc9948e282743e54c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75364676, jitterRate=0.1230211853981018}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:48,981 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0267affe6d2f53bbc9948e282743e54c: Writing region info on filesystem at 1732376808971Initializing all the Stores at 1732376808972 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808972Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808973 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376808973Cleaning up temporary data from old regions at 1732376808979 (+6 ms)Region opened successfully at 1732376808981 (+2 ms) 2024-11-23T15:46:48,988 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0267affe6d2f53bbc9948e282743e54c 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-23T15:46:49,003 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/a/02b70d49209247b3b4ff3167387a2b3c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732376808981/Put/seqid=0 2024-11-23T15:46:49,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741933_1112 (size=5958) 2024-11-23T15:46:49,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741933_1112 (size=5958) 2024-11-23T15:46:49,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741933_1112 (size=5958) 2024-11-23T15:46:49,010 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/a/02b70d49209247b3b4ff3167387a2b3c 2024-11-23T15:46:49,016 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/a/02b70d49209247b3b4ff3167387a2b3c as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/a/02b70d49209247b3b4ff3167387a2b3c 2024-11-23T15:46:49,021 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/a/02b70d49209247b3b4ff3167387a2b3c, entries=10, sequenceid=13, filesize=5.8 K 2024-11-23T15:46:49,022 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 0267affe6d2f53bbc9948e282743e54c in 34ms, sequenceid=13, compaction requested=false 2024-11-23T15:46:49,022 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0267affe6d2f53bbc9948e282743e54c: 2024-11-23T15:46:49,036 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0267affe6d2f53bbc9948e282743e54c, disabling compactions & flushes 2024-11-23T15:46:49,036 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:49,036 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:49,036 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. after waiting 0 ms 2024-11-23T15:46:49,036 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:49,037 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:49,037 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:49,037 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0267affe6d2f53bbc9948e282743e54c: Waiting for close lock at 1732376809036Disabling compacts and flushes for region at 1732376809036Disabling writes for close at 1732376809036Writing region close event to WAL at 1732376809037 (+1 ms)Closed at 1732376809037 2024-11-23T15:46:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741932_1111 (size=6243) 2024-11-23T15:46:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741932_1111 (size=6243) 2024-11-23T15:46:49,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741932_1111 (size=6243) 2024-11-23T15:46:49,060 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924, size=6.1 K (6243bytes) 2024-11-23T15:46:49,060 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924 2024-11-23T15:46:49,061 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924 after 1ms 2024-11-23T15:46:49,087 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:49,087 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924 took 27ms 2024-11-23T15:46:49,089 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924 so closing down 2024-11-23T15:46:49,089 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:49,090 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732376808924.temp 2024-11-23T15:46:49,114 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000003-wal.1732376808924.temp 2024-11-23T15:46:49,114 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741934_1113 (size=5575) 2024-11-23T15:46:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741934_1113 (size=5575) 2024-11-23T15:46:49,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741934_1113 (size=5575) 2024-11-23T15:46:49,121 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000003-wal.1732376808924.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:49,122 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000003-wal.1732376808924.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000035 2024-11-23T15:46:49,122 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-23T15:46:49,122 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924, journal: Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924, size=6.1 K (6243bytes) at 1732376809060Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924 so closing down at 1732376809089 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000003-wal.1732376808924.temp at 1732376809114 (+25 ms)3 split writer threads finished at 1732376809114Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000003-wal.1732376808924.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732376809121 (+7 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000003-wal.1732376808924.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000035 at 1732376809122 (+1 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732376809122 2024-11-23T15:46:49,123 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376808924 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376808924 2024-11-23T15:46:49,124 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000035 2024-11-23T15:46:49,124 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:49,125 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:49,144 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126, exclude list is [], retry=0 2024-11-23T15:46:49,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:49,147 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:49,148 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:49,173 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 2024-11-23T15:46:49,173 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36223:36223),(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785)] 2024-11-23T15:46:49,173 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0267affe6d2f53bbc9948e282743e54c, NAME => 'testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:46:49,174 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:49,174 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,174 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,175 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,176 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName a 2024-11-23T15:46:49,176 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:49,182 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/a/02b70d49209247b3b4ff3167387a2b3c 2024-11-23T15:46:49,182 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:49,182 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,183 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName b 2024-11-23T15:46:49,183 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:49,183 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:49,183 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,184 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName c 2024-11-23T15:46:49,184 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:49,185 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:49,185 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,187 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,188 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000035 2024-11-23T15:46:49,214 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:49,216 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000035 2024-11-23T15:46:49,216 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0267affe6d2f53bbc9948e282743e54c 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-23T15:46:49,229 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/b/7b7dab47becb4335ab2c57994ac8ee1b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732376809022/Put/seqid=0 2024-11-23T15:46:49,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741936_1115 (size=5958) 2024-11-23T15:46:49,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741936_1115 (size=5958) 2024-11-23T15:46:49,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741936_1115 (size=5958) 2024-11-23T15:46:49,235 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/b/7b7dab47becb4335ab2c57994ac8ee1b 2024-11-23T15:46:49,253 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/c/e61d30fc1f1a4e6c910dcf1cd6e0289a is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732376809028/Put/seqid=0 2024-11-23T15:46:49,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741937_1116 (size=5958) 2024-11-23T15:46:49,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741937_1116 (size=5958) 2024-11-23T15:46:49,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741937_1116 (size=5958) 2024-11-23T15:46:49,260 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/c/e61d30fc1f1a4e6c910dcf1cd6e0289a 2024-11-23T15:46:49,265 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/b/7b7dab47becb4335ab2c57994ac8ee1b as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/b/7b7dab47becb4335ab2c57994ac8ee1b 2024-11-23T15:46:49,270 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/b/7b7dab47becb4335ab2c57994ac8ee1b, entries=10, sequenceid=35, filesize=5.8 K 2024-11-23T15:46:49,271 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/c/e61d30fc1f1a4e6c910dcf1cd6e0289a as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/c/e61d30fc1f1a4e6c910dcf1cd6e0289a 2024-11-23T15:46:49,276 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/c/e61d30fc1f1a4e6c910dcf1cd6e0289a, entries=10, sequenceid=35, filesize=5.8 K 2024-11-23T15:46:49,276 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 0267affe6d2f53bbc9948e282743e54c in 60ms, sequenceid=35, compaction requested=false; wal=null 2024-11-23T15:46:49,276 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000035 2024-11-23T15:46:49,277 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,277 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,278 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:49,280 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:49,282 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-23T15:46:49,283 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0267affe6d2f53bbc9948e282743e54c; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63973991, jitterRate=-0.046713247895240784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:49,283 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0267affe6d2f53bbc9948e282743e54c: Writing region info on filesystem at 1732376809174Initializing all the Stores at 1732376809175 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376809175Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376809175Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376809175Obtaining lock to block concurrent updates at 1732376809216 (+41 ms)Preparing flush snapshotting stores in 0267affe6d2f53bbc9948e282743e54c at 1732376809216Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732376809216Flushing stores of testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. at 1732376809216Flushing 0267affe6d2f53bbc9948e282743e54c/b: creating writer at 1732376809216Flushing 0267affe6d2f53bbc9948e282743e54c/b: appending metadata at 1732376809228 (+12 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/b: closing flushed file at 1732376809228Flushing 0267affe6d2f53bbc9948e282743e54c/c: creating writer at 1732376809239 (+11 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/c: appending metadata at 1732376809253 (+14 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/c: closing flushed file at 1732376809253Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ea1e757: reopening flushed file at 1732376809264 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66bf1106: reopening flushed file at 1732376809270 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 0267affe6d2f53bbc9948e282743e54c in 60ms, sequenceid=35, compaction requested=false; wal=null at 1732376809276 (+6 ms)Cleaning up temporary data from old regions at 1732376809278 (+2 ms)Region opened successfully at 1732376809283 (+5 ms) 2024-11-23T15:46:49,357 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126, size=0 (0bytes) 2024-11-23T15:46:49,357 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 might be still open, length is 0 2024-11-23T15:46:49,358 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 2024-11-23T15:46:49,358 WARN [IPC Server handler 2 on default port 33345 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-23T15:46:49,358 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 after 0ms 2024-11-23T15:46:49,580 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:56688 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:44917:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56688 dst: /127.0.0.1:44917 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44917 remote=/127.0.0.1:56688]. Total timeout mills is 60000, 59735 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:49,582 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:44508 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37735:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44508 dst: /127.0.0.1:37735 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:49,582 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1639012755_22 at /127.0.0.1:60810 [Receiving block BP-1981799117-172.17.0.2-1732376767483:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:35471:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60810 dst: /127.0.0.1:35471 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:49,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741935_1117 (size=5567) 2024-11-23T15:46:49,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741935_1117 (size=5567) 2024-11-23T15:46:51,817 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-23T15:46:51,817 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:51,818 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-23T15:46:51,818 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-23T15:46:53,359 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 after 4001ms 2024-11-23T15:46:53,396 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:53,396 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 took 4039ms 2024-11-23T15:46:53,398 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126; continuing. 2024-11-23T15:46:53,398 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 so closing down 2024-11-23T15:46:53,398 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-23T15:46:53,399 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732376809126.temp 2024-11-23T15:46:53,427 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000037-wal.1732376809126.temp 2024-11-23T15:46:53,427 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-23T15:46:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741938_1118 (size=5575) 2024-11-23T15:46:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741938_1118 (size=5575) 2024-11-23T15:46:53,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741938_1118 (size=5575) 2024-11-23T15:46:53,435 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000037-wal.1732376809126.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-23T15:46:53,436 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000037-wal.1732376809126.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000066 2024-11-23T15:46:53,436 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126, size=0, length=0, corrupted=false, cancelled=false 2024-11-23T15:46:53,436 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126, journal: Splitting hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126, size=0 (0bytes) at 1732376809357Finishing writing output for hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 so closing down at 1732376813398 (+4041 ms)Creating recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000037-wal.1732376809126.temp at 1732376813427 (+29 ms)3 split writer threads finished at 1732376813427Closed recovered edits writer path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000037-wal.1732376809126.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732376813435 (+8 ms)Rename recovered edits hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000037-wal.1732376809126.temp to hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000066 at 1732376813436 (+1 ms)Processed 30 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126, size=0, length=0, corrupted=false, cancelled=false at 1732376813436 2024-11-23T15:46:53,438 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 to hdfs://localhost:33345/hbase/oldWALs/wal.1732376809126 2024-11-23T15:46:53,439 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000066 2024-11-23T15:46:53,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-23T15:46:53,441 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33345/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850, archiveDir=hdfs://localhost:33345/hbase/oldWALs, maxLogs=32 2024-11-23T15:46:53,453 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376813441, exclude list is [], retry=0 2024-11-23T15:46:53,455 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35471,DS-b10ac704-f88d-415d-b341-90745ac2302f,DISK] 2024-11-23T15:46:53,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37735,DS-2a156afc-b4f3-409e-9d84-2bfc622042ac,DISK] 2024-11-23T15:46:53,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44917,DS-020d36d3-1e4b-42de-87db-071fd0edcd95,DISK] 2024-11-23T15:46:53,480 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376813441 2024-11-23T15:46:53,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37309:37309),(127.0.0.1/127.0.0.1:44785:44785),(127.0.0.1/127.0.0.1:36223:36223)] 2024-11-23T15:46:53,481 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:46:53,482 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,483 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName a 2024-11-23T15:46:53,483 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:53,488 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/a/02b70d49209247b3b4ff3167387a2b3c 2024-11-23T15:46:53,488 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:53,488 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,489 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName b 2024-11-23T15:46:53,489 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:53,494 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/b/7b7dab47becb4335ab2c57994ac8ee1b 2024-11-23T15:46:53,494 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:53,494 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,495 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0267affe6d2f53bbc9948e282743e54c columnFamilyName c 2024-11-23T15:46:53,495 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:46:53,499 DEBUG [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/c/e61d30fc1f1a4e6c910dcf1cd6e0289a 2024-11-23T15:46:53,499 INFO [StoreOpener-0267affe6d2f53bbc9948e282743e54c-1 {}] regionserver.HStore(327): Store=0267affe6d2f53bbc9948e282743e54c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:46:53,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,502 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000066 2024-11-23T15:46:53,526 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-23T15:46:53,529 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000066 2024-11-23T15:46:53,530 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0267affe6d2f53bbc9948e282743e54c 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-23T15:46:53,543 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/a/b0751d5da967443f8f60b851a30c411a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732376809290/Put/seqid=0 2024-11-23T15:46:53,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741940_1120 (size=5958) 2024-11-23T15:46:53,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741940_1120 (size=5958) 2024-11-23T15:46:53,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741940_1120 (size=5958) 2024-11-23T15:46:53,550 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/a/b0751d5da967443f8f60b851a30c411a 2024-11-23T15:46:53,575 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/b/09519c9932214c708df2c02e50c8cfe7 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732376809299/Put/seqid=0 2024-11-23T15:46:53,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741941_1121 (size=5958) 2024-11-23T15:46:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741941_1121 (size=5958) 2024-11-23T15:46:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741941_1121 (size=5958) 2024-11-23T15:46:53,588 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/b/09519c9932214c708df2c02e50c8cfe7 2024-11-23T15:46:53,606 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/c/3049066ac0304dadbea5cacd1e8bcdf6 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732376809306/Put/seqid=0 2024-11-23T15:46:53,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741942_1122 (size=5958) 2024-11-23T15:46:53,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741942_1122 (size=5958) 2024-11-23T15:46:53,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741942_1122 (size=5958) 2024-11-23T15:46:53,612 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/c/3049066ac0304dadbea5cacd1e8bcdf6 2024-11-23T15:46:53,618 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/a/b0751d5da967443f8f60b851a30c411a as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/a/b0751d5da967443f8f60b851a30c411a 2024-11-23T15:46:53,622 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/a/b0751d5da967443f8f60b851a30c411a, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T15:46:53,623 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/b/09519c9932214c708df2c02e50c8cfe7 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/b/09519c9932214c708df2c02e50c8cfe7 2024-11-23T15:46:53,627 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/b/09519c9932214c708df2c02e50c8cfe7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T15:46:53,628 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/.tmp/c/3049066ac0304dadbea5cacd1e8bcdf6 as hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/c/3049066ac0304dadbea5cacd1e8bcdf6 2024-11-23T15:46:53,632 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/c/3049066ac0304dadbea5cacd1e8bcdf6, entries=10, sequenceid=66, filesize=5.8 K 2024-11-23T15:46:53,633 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 0267affe6d2f53bbc9948e282743e54c in 104ms, sequenceid=66, compaction requested=false; wal=null 2024-11-23T15:46:53,633 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/0000000000000000066 2024-11-23T15:46:53,634 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,634 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,635 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-23T15:46:53,636 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0267affe6d2f53bbc9948e282743e54c 2024-11-23T15:46:53,638 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/hbase/data/default/testReplayEditsWrittenViaHRegion/0267affe6d2f53bbc9948e282743e54c/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-23T15:46:53,639 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0267affe6d2f53bbc9948e282743e54c; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60728252, jitterRate=-0.09507852792739868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-23T15:46:53,639 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0267affe6d2f53bbc9948e282743e54c: Writing region info on filesystem at 1732376813481Initializing all the Stores at 1732376813482 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376813482Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376813482Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376813482Obtaining lock to block concurrent updates at 1732376813530 (+48 ms)Preparing flush snapshotting stores in 0267affe6d2f53bbc9948e282743e54c at 1732376813530Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732376813530Flushing stores of testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. at 1732376813530Flushing 0267affe6d2f53bbc9948e282743e54c/a: creating writer at 1732376813530Flushing 0267affe6d2f53bbc9948e282743e54c/a: appending metadata at 1732376813542 (+12 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/a: closing flushed file at 1732376813542Flushing 0267affe6d2f53bbc9948e282743e54c/b: creating writer at 1732376813555 (+13 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/b: appending metadata at 1732376813575 (+20 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/b: closing flushed file at 1732376813575Flushing 0267affe6d2f53bbc9948e282743e54c/c: creating writer at 1732376813592 (+17 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/c: appending metadata at 1732376813605 (+13 ms)Flushing 0267affe6d2f53bbc9948e282743e54c/c: closing flushed file at 1732376813605Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@244612c9: reopening flushed file at 1732376813617 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5effa1ae: reopening flushed file at 1732376813622 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9357e3b: reopening flushed file at 1732376813627 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 0267affe6d2f53bbc9948e282743e54c in 104ms, sequenceid=66, compaction requested=false; wal=null at 1732376813633 (+6 ms)Cleaning up temporary data from old regions at 1732376813634 (+1 ms)Region opened successfully at 1732376813639 (+5 ms) 2024-11-23T15:46:53,651 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0267affe6d2f53bbc9948e282743e54c, disabling compactions & flushes 2024-11-23T15:46:53,651 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:53,651 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:53,651 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. after waiting 0 ms 2024-11-23T15:46:53,651 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:53,652 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732376808852.0267affe6d2f53bbc9948e282743e54c. 2024-11-23T15:46:53,652 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0267affe6d2f53bbc9948e282743e54c: Waiting for close lock at 1732376813651Disabling compacts and flushes for region at 1732376813651Disabling writes for close at 1732376813651Writing region close event to WAL at 1732376813652 (+1 ms)Closed at 1732376813652 2024-11-23T15:46:53,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741939_1119 (size=175) 2024-11-23T15:46:53,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741939_1119 (size=175) 2024-11-23T15:46:53,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741939_1119 (size=175) 2024-11-23T15:46:53,657 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-23T15:46:53,657 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732376813441) 2024-11-23T15:46:53,670 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=448 (was 443) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33345 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_306214409_22 at /127.0.0.1:36140 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_306214409_22 at /127.0.0.1:48902 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:33345 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_306214409_22 at /127.0.0.1:56408 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 247) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7536 (was 7657) 2024-11-23T15:46:53,671 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-23T15:46:53,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T15:46:53,671 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:46:53,671 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:46:53,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:53,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:53,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:46:53,672 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T15:46:53,672 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=600130957, stopped=false 2024-11-23T15:46:53,672 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b712f9af2c12,34021,1732376771486 2024-11-23T15:46:53,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T15:46:53,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T15:46:53,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T15:46:53,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:53,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:53,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:53,691 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T15:46:53,691 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:46:53,691 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:46:53,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:53,691 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b712f9af2c12,34281,1732376772315' ***** 2024-11-23T15:46:53,691 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T15:46:53,691 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b712f9af2c12,40435,1732376772380' ***** 2024-11-23T15:46:53,691 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T15:46:53,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:46:53,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T15:46:53,692 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T15:46:53,692 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(959): stopping server b712f9af2c12,40435,1732376772380 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:46:53,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b712f9af2c12:40435. 2024-11-23T15:46:53,692 INFO [RS:1;b712f9af2c12:34281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T15:46:53,692 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T15:46:53,692 DEBUG [RS:2;b712f9af2c12:40435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:46:53,692 INFO [RS:1;b712f9af2c12:34281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T15:46:53,692 DEBUG [RS:2;b712f9af2c12:40435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:53,692 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(3091): Received CLOSE for 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T15:46:53,692 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(959): stopping server b712f9af2c12,34281,1732376772315 2024-11-23T15:46:53,692 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T15:46:53,692 INFO [RS:1;b712f9af2c12:34281 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:46:53,693 INFO [RS:1;b712f9af2c12:34281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b712f9af2c12:34281. 2024-11-23T15:46:53,693 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6c6b84da1fb3ebdb0344eecf253fc022, disabling compactions & flushes 2024-11-23T15:46:53,693 DEBUG [RS:1;b712f9af2c12:34281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:46:53,693 DEBUG [RS:1;b712f9af2c12:34281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:53,693 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. after waiting 0 ms 2024-11-23T15:46:53,693 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:53,693 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1325): Online Regions={6c6b84da1fb3ebdb0344eecf253fc022=testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022.} 2024-11-23T15:46:53,693 DEBUG [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1351): Waiting on 6c6b84da1fb3ebdb0344eecf253fc022 2024-11-23T15:46:53,693 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T15:46:53,693 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T15:46:53,693 DEBUG [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T15:46:53,693 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T15:46:53,693 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T15:46:53,693 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-23T15:46:53,697 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6c6b84da1fb3ebdb0344eecf253fc022/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-23T15:46:53,697 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:53,698 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6c6b84da1fb3ebdb0344eecf253fc022: Waiting for close lock at 1732376813692Running coprocessor pre-close hooks at 1732376813693 (+1 ms)Disabling compacts and flushes for region at 1732376813693Disabling writes for close at 1732376813693Writing region close event to WAL at 1732376813694 (+1 ms)Running coprocessor post-close hooks at 1732376813697 (+3 ms)Closed at 1732376813697 2024-11-23T15:46:53,698 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022. 2024-11-23T15:46:53,702 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/info/c031bb528079456d88581591a8fc2a97 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732376788545.6c6b84da1fb3ebdb0344eecf253fc022./info:regioninfo/1732376792768/Put/seqid=0 2024-11-23T15:46:53,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741943_1123 (size=6778) 2024-11-23T15:46:53,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741943_1123 (size=6778) 2024-11-23T15:46:53,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741943_1123 (size=6778) 2024-11-23T15:46:53,711 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/info/c031bb528079456d88581591a8fc2a97 2024-11-23T15:46:53,717 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/.tmp/info/c031bb528079456d88581591a8fc2a97 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/info/c031bb528079456d88581591a8fc2a97 2024-11-23T15:46:53,722 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/info/c031bb528079456d88581591a8fc2a97, entries=8, sequenceid=23, filesize=6.6 K 2024-11-23T15:46:53,723 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=23, compaction requested=false 2024-11-23T15:46:53,727 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-23T15:46:53,728 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:46:53,728 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T15:46:53,728 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732376813693Running coprocessor pre-close hooks at 1732376813693Disabling compacts and flushes for region at 1732376813693Disabling writes for close at 1732376813693Obtaining lock to block concurrent updates at 1732376813693Preparing flush snapshotting stores in 1588230740 at 1732376813693Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732376813694 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732376813699 (+5 ms)Flushing 1588230740/info: creating writer at 1732376813699Flushing 1588230740/info: appending metadata at 1732376813702 (+3 ms)Flushing 1588230740/info: closing flushed file at 1732376813702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d7d87a3: reopening flushed file at 1732376813716 (+14 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=23, compaction requested=false at 1732376813723 (+7 ms)Writing region close event to WAL at 1732376813724 (+1 ms)Running coprocessor post-close hooks at 1732376813728 (+4 ms)Closed at 1732376813728 2024-11-23T15:46:53,728 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T15:46:53,774 INFO [regionserver/b712f9af2c12:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:46:53,774 INFO [regionserver/b712f9af2c12:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:46:53,811 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T15:46:53,811 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T15:46:53,882 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T15:46:53,882 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T15:46:53,893 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(976): stopping server b712f9af2c12,34281,1732376772315; all regions closed. 2024-11-23T15:46:53,893 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(976): stopping server b712f9af2c12,40435,1732376772380; all regions closed. 2024-11-23T15:46:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741835_1011 (size=3621) 2024-11-23T15:46:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741893_1071 (size=2625) 2024-11-23T15:46:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741835_1011 (size=3621) 2024-11-23T15:46:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741893_1071 (size=2625) 2024-11-23T15:46:53,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741835_1011 (size=3621) 2024-11-23T15:46:53,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741893_1071 (size=2625) 2024-11-23T15:46:53,899 DEBUG [RS:1;b712f9af2c12:34281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs 2024-11-23T15:46:53,899 INFO [RS:1;b712f9af2c12:34281 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b712f9af2c12%2C34281%2C1732376772315:(num 1732376773966) 2024-11-23T15:46:53,899 DEBUG [RS:2;b712f9af2c12:40435 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs 2024-11-23T15:46:53,899 DEBUG [RS:1;b712f9af2c12:34281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:53,899 INFO [RS:2;b712f9af2c12:40435 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b712f9af2c12%2C40435%2C1732376772380.meta:.meta(num 1732376791807) 2024-11-23T15:46:53,899 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:46:53,899 INFO [RS:1;b712f9af2c12:34281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:46:53,900 INFO [RS:1;b712f9af2c12:34281 {}] hbase.ChoreService(370): Chore service for: regionserver/b712f9af2c12:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T15:46:53,900 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T15:46:53,900 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T15:46:53,900 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T15:46:53,900 INFO [RS:1;b712f9af2c12:34281 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:46:53,900 INFO [RS:1;b712f9af2c12:34281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34281 2024-11-23T15:46:53,900 INFO [regionserver/b712f9af2c12:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:46:53,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741834_1010 (size=175) 2024-11-23T15:46:53,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b712f9af2c12,34281,1732376772315 2024-11-23T15:46:53,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741834_1010 (size=175) 2024-11-23T15:46:53,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741834_1010 (size=175) 2024-11-23T15:46:53,908 INFO [RS:1;b712f9af2c12:34281 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:46:53,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:46:53,908 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/WALs/b712f9af2c12,40435,1732376772380/b712f9af2c12%2C40435%2C1732376772380.1732376773966 not finished, retry = 0 2024-11-23T15:46:53,909 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b712f9af2c12,34281,1732376772315] 2024-11-23T15:46:53,924 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b712f9af2c12,34281,1732376772315 already deleted, retry=false 2024-11-23T15:46:53,924 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b712f9af2c12,34281,1732376772315 expired; onlineServers=1 2024-11-23T15:46:54,011 DEBUG [RS:2;b712f9af2c12:40435 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/oldWALs 2024-11-23T15:46:54,011 INFO [RS:2;b712f9af2c12:40435 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b712f9af2c12%2C40435%2C1732376772380:(num 1732376773966) 2024-11-23T15:46:54,011 DEBUG [RS:2;b712f9af2c12:40435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:46:54,011 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:46:54,011 INFO [RS:2;b712f9af2c12:40435 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:46:54,012 INFO [RS:2;b712f9af2c12:40435 {}] hbase.ChoreService(370): Chore service for: regionserver/b712f9af2c12:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T15:46:54,012 INFO [RS:2;b712f9af2c12:40435 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:46:54,012 INFO [regionserver/b712f9af2c12:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:46:54,012 INFO [RS:2;b712f9af2c12:40435 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40435 2024-11-23T15:46:54,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:54,016 INFO [RS:1;b712f9af2c12:34281 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:46:54,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34281-0x10169a94b600002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:54,016 INFO [RS:1;b712f9af2c12:34281 {}] regionserver.HRegionServer(1031): Exiting; stopping=b712f9af2c12,34281,1732376772315; zookeeper connection closed. 2024-11-23T15:46:54,016 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@72770aa4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@72770aa4 2024-11-23T15:46:54,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b712f9af2c12,40435,1732376772380 2024-11-23T15:46:54,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:46:54,032 INFO [RS:2;b712f9af2c12:40435 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:46:54,033 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b712f9af2c12,40435,1732376772380] 2024-11-23T15:46:54,059 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b712f9af2c12,40435,1732376772380 already deleted, retry=false 2024-11-23T15:46:54,059 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b712f9af2c12,40435,1732376772380 expired; onlineServers=0 2024-11-23T15:46:54,060 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b712f9af2c12,34021,1732376771486' ***** 2024-11-23T15:46:54,060 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T15:46:54,060 INFO [M:0;b712f9af2c12:34021 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:46:54,060 INFO [M:0;b712f9af2c12:34021 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:46:54,060 DEBUG [M:0;b712f9af2c12:34021 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T15:46:54,060 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T15:46:54,060 DEBUG [M:0;b712f9af2c12:34021 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T15:46:54,060 DEBUG [master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732376773596 {}] cleaner.HFileCleaner(306): Exit Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732376773596,5,FailOnTimeoutGroup] 2024-11-23T15:46:54,060 DEBUG [master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732376773597 {}] cleaner.HFileCleaner(306): Exit Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732376773597,5,FailOnTimeoutGroup] 2024-11-23T15:46:54,061 INFO [M:0;b712f9af2c12:34021 {}] hbase.ChoreService(370): Chore service for: master/b712f9af2c12:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T15:46:54,061 INFO [M:0;b712f9af2c12:34021 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:46:54,061 DEBUG [M:0;b712f9af2c12:34021 {}] master.HMaster(1795): Stopping service threads 2024-11-23T15:46:54,061 INFO [M:0;b712f9af2c12:34021 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T15:46:54,061 INFO [M:0;b712f9af2c12:34021 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T15:46:54,062 INFO [M:0;b712f9af2c12:34021 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T15:46:54,062 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T15:46:54,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T15:46:54,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:46:54,065 DEBUG [M:0;b712f9af2c12:34021 {}] zookeeper.ZKUtil(347): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T15:46:54,065 WARN [M:0;b712f9af2c12:34021 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T15:46:54,066 INFO [M:0;b712f9af2c12:34021 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/.lastflushedseqids 2024-11-23T15:46:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741944_1124 (size=119) 2024-11-23T15:46:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741944_1124 (size=119) 2024-11-23T15:46:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741944_1124 (size=119) 2024-11-23T15:46:54,078 INFO [M:0;b712f9af2c12:34021 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T15:46:54,078 INFO [M:0;b712f9af2c12:34021 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T15:46:54,078 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T15:46:54,078 INFO [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:54,079 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:54,079 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T15:46:54,079 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:54,079 INFO [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-11-23T15:46:54,096 DEBUG [M:0;b712f9af2c12:34021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f39ee59521c2478c883bc21170bcc21b is 82, key is hbase:meta,,1/info:regioninfo/1732376792031/Put/seqid=0 2024-11-23T15:46:54,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741945_1125 (size=6063) 2024-11-23T15:46:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741945_1125 (size=6063) 2024-11-23T15:46:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741945_1125 (size=6063) 2024-11-23T15:46:54,106 INFO [M:0;b712f9af2c12:34021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f39ee59521c2478c883bc21170bcc21b 2024-11-23T15:46:54,126 DEBUG [M:0;b712f9af2c12:34021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8accde908e2e4fb6a1d5d37653dc2899 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732376788975/Put/seqid=0 2024-11-23T15:46:54,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741946_1126 (size=7906) 2024-11-23T15:46:54,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741946_1126 (size=7906) 2024-11-23T15:46:54,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741946_1126 (size=7906) 2024-11-23T15:46:54,134 INFO [M:0;b712f9af2c12:34021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8accde908e2e4fb6a1d5d37653dc2899 2024-11-23T15:46:54,139 INFO [M:0;b712f9af2c12:34021 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8accde908e2e4fb6a1d5d37653dc2899 2024-11-23T15:46:54,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:54,149 INFO [RS:2;b712f9af2c12:40435 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:46:54,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40435-0x10169a94b600003, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:54,149 INFO [RS:2;b712f9af2c12:40435 {}] regionserver.HRegionServer(1031): Exiting; stopping=b712f9af2c12,40435,1732376772380; zookeeper connection closed. 2024-11-23T15:46:54,149 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@347b2440 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@347b2440 2024-11-23T15:46:54,150 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T15:46:54,155 DEBUG [M:0;b712f9af2c12:34021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/82ff900596d6474f8337a59f3084d2be is 69, key is b712f9af2c12,34281,1732376772315/rs:state/1732376773679/Put/seqid=0 2024-11-23T15:46:54,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741947_1127 (size=5440) 2024-11-23T15:46:54,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741947_1127 (size=5440) 2024-11-23T15:46:54,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741947_1127 (size=5440) 2024-11-23T15:46:54,164 INFO [M:0;b712f9af2c12:34021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/82ff900596d6474f8337a59f3084d2be 2024-11-23T15:46:54,168 INFO [M:0;b712f9af2c12:34021 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 82ff900596d6474f8337a59f3084d2be 2024-11-23T15:46:54,170 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f39ee59521c2478c883bc21170bcc21b as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f39ee59521c2478c883bc21170bcc21b 2024-11-23T15:46:54,175 INFO [M:0;b712f9af2c12:34021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f39ee59521c2478c883bc21170bcc21b, entries=14, sequenceid=207, filesize=5.9 K 2024-11-23T15:46:54,176 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8accde908e2e4fb6a1d5d37653dc2899 as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8accde908e2e4fb6a1d5d37653dc2899 2024-11-23T15:46:54,181 INFO [M:0;b712f9af2c12:34021 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8accde908e2e4fb6a1d5d37653dc2899 2024-11-23T15:46:54,181 INFO [M:0;b712f9af2c12:34021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8accde908e2e4fb6a1d5d37653dc2899, entries=21, sequenceid=207, filesize=7.7 K 2024-11-23T15:46:54,182 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/82ff900596d6474f8337a59f3084d2be as hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/82ff900596d6474f8337a59f3084d2be 2024-11-23T15:46:54,187 INFO [M:0;b712f9af2c12:34021 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 82ff900596d6474f8337a59f3084d2be 2024-11-23T15:46:54,187 INFO [M:0;b712f9af2c12:34021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33345/user/jenkins/test-data/e25bf9d4-4d1f-be38-7720-cad436f00748/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/82ff900596d6474f8337a59f3084d2be, entries=3, sequenceid=207, filesize=5.3 K 2024-11-23T15:46:54,189 INFO [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=207, compaction requested=false 2024-11-23T15:46:54,191 INFO [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:46:54,191 DEBUG [M:0;b712f9af2c12:34021 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732376814078Disabling compacts and flushes for region at 1732376814078Disabling writes for close at 1732376814079 (+1 ms)Obtaining lock to block concurrent updates at 1732376814079Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732376814079Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1732376814079Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732376814080 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732376814080Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732376814096 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732376814096Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732376814111 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732376814125 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732376814125Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732376814139 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732376814154 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732376814154Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dd9d7c6: reopening flushed file at 1732376814169 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1418ec82: reopening flushed file at 1732376814175 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b0997ce: reopening flushed file at 1732376814181 (+6 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=207, compaction requested=false at 1732376814189 (+8 ms)Writing region close event to WAL at 1732376814191 (+2 ms)Closed at 1732376814191 2024-11-23T15:46:54,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37735 is added to blk_1073741830_1006 (size=102150) 2024-11-23T15:46:54,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35471 is added to blk_1073741830_1006 (size=102150) 2024-11-23T15:46:54,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44917 is added to blk_1073741830_1006 (size=102150) 2024-11-23T15:46:54,195 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:46:54,195 INFO [M:0;b712f9af2c12:34021 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T15:46:54,195 INFO [M:0;b712f9af2c12:34021 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34021 2024-11-23T15:46:54,195 INFO [M:0;b712f9af2c12:34021 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:46:54,306 INFO [M:0;b712f9af2c12:34021 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:46:54,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:54,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34021-0x10169a94b600000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:46:54,313 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376803286 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376803286 (inode 16688) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732376793936/wal.1732376803286 (inode 16688) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T15:46:54,316 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793751 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793751 (inode 16665) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732376793405/wal.1732376793751 (inode 16665) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T15:46:54,317 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732376784003/wal.1732376784133 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:54,320 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376804100 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376804100 (inode 16714) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732376803536/wal.1732376804100 (inode 16714) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T15:46:54,320 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732376804283/wal.1732376804357 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:54,321 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732376808850/wal.1732376809126 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:54,321 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732376776146/wal.1732376776241 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:46:54,324 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal.1732376775967 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal.1732376775967 (inode 16485) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732376775597/wal.1732376775967 (inode 16485) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T15:46:54,326 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal.1732376775263 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal.1732376775263 (inode 16462) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732376774825/wal.1732376775263 (inode 16462) Holder DFSClient_NONMAPREDUCE_1639012755_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-23T15:46:54,331 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@773b4a40{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:46:54,334 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:46:54,334 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:46:54,334 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:46:54,334 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,STOPPED} 2024-11-23T15:46:54,337 WARN [BP-1981799117-172.17.0.2-1732376767483 heartbeating to localhost/127.0.0.1:33345 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T15:46:54,337 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T15:46:54,337 WARN [BP-1981799117-172.17.0.2-1732376767483 heartbeating to localhost/127.0.0.1:33345 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1981799117-172.17.0.2-1732376767483 (Datanode Uuid ed3db639-a30a-4375-bc03-437a25bdabfc) service to localhost/127.0.0.1:33345 2024-11-23T15:46:54,337 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T15:46:54,338 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data5/current/BP-1981799117-172.17.0.2-1732376767483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:46:54,339 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data6/current/BP-1981799117-172.17.0.2-1732376767483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:46:54,339 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T15:46:54,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75d96791{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:46:54,342 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:46:54,342 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:46:54,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:46:54,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,STOPPED} 2024-11-23T15:46:54,343 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T15:46:54,343 WARN [BP-1981799117-172.17.0.2-1732376767483 heartbeating to localhost/127.0.0.1:33345 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T15:46:54,343 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T15:46:54,343 WARN [BP-1981799117-172.17.0.2-1732376767483 heartbeating to localhost/127.0.0.1:33345 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1981799117-172.17.0.2-1732376767483 (Datanode Uuid 586a75e0-c289-4b4c-be4c-3563ed9c80a5) service to localhost/127.0.0.1:33345 2024-11-23T15:46:54,344 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data3/current/BP-1981799117-172.17.0.2-1732376767483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:46:54,344 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data4/current/BP-1981799117-172.17.0.2-1732376767483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:46:54,344 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T15:46:54,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d480c14{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:46:54,347 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:46:54,347 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:46:54,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:46:54,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,STOPPED} 2024-11-23T15:46:54,349 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T15:46:54,349 WARN [BP-1981799117-172.17.0.2-1732376767483 heartbeating to localhost/127.0.0.1:33345 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T15:46:54,349 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T15:46:54,349 WARN [BP-1981799117-172.17.0.2-1732376767483 heartbeating to localhost/127.0.0.1:33345 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1981799117-172.17.0.2-1732376767483 (Datanode Uuid 8f49f37a-b0c1-4fe8-845f-47fc00815dc3) service to localhost/127.0.0.1:33345 2024-11-23T15:46:54,350 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data1/current/BP-1981799117-172.17.0.2-1732376767483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:46:54,350 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/cluster_c9050aa5-7511-1c7d-bf22-236c042a569d/data/data2/current/BP-1981799117-172.17.0.2-1732376767483 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:46:54,350 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T15:46:54,355 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14f32243{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T15:46:54,356 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:46:54,356 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:46:54,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:46:54,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/173958b4-6bf2-c5ce-8216-7568a6b31cfc/hadoop.log.dir/,STOPPED} 2024-11-23T15:46:54,365 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T15:46:54,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down