2024-11-13 05:45:05,000 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 05:45:05,011 main DEBUG Took 0.008898 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 05:45:05,011 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 05:45:05,011 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 05:45:05,012 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 05:45:05,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,019 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 05:45:05,030 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,032 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,033 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,034 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,035 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,036 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,037 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,037 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,037 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,038 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,039 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,039 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,040 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,040 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,040 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,041 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:45:05,041 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,041 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 05:45:05,043 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:45:05,045 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 05:45:05,047 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 05:45:05,047 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 05:45:05,048 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 05:45:05,049 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 05:45:05,059 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 05:45:05,062 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 05:45:05,065 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 05:45:05,065 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 05:45:05,066 main DEBUG createAppenders(={Console}) 2024-11-13 05:45:05,067 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-13 05:45:05,067 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 05:45:05,067 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-13 05:45:05,068 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 05:45:05,068 main DEBUG OutputStream closed 2024-11-13 05:45:05,069 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 05:45:05,069 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 05:45:05,069 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-13 05:45:05,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 05:45:05,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 05:45:05,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 05:45:05,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 05:45:05,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 05:45:05,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 05:45:05,167 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 05:45:05,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 05:45:05,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 05:45:05,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 05:45:05,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 05:45:05,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 05:45:05,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 05:45:05,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 05:45:05,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 05:45:05,170 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 05:45:05,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 05:45:05,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 05:45:05,173 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 05:45:05,174 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-13 05:45:05,174 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 05:45:05,174 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-13T05:45:05,417 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a 2024-11-13 05:45:05,420 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 05:45:05,421 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T05:45:05,429 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T05:45:05,436 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-13T05:45:05,463 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T05:45:05,510 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T05:45:05,510 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T05:45:05,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:45:05,541 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4, deleteOnExit=true 2024-11-13T05:45:05,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:45:05,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/test.cache.data in system properties and HBase conf 2024-11-13T05:45:05,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:45:05,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:45:05,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:45:05,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:45:05,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:45:05,616 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T05:45:05,703 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:45:05,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:45:05,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:45:05,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:45:05,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:45:05,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:45:05,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:45:05,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:45:05,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:45:05,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:45:05,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:45:05,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:45:05,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:45:05,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:45:05,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:45:06,701 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T05:45:06,768 INFO [Time-limited test {}] log.Log(170): Logging initialized @2447ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T05:45:06,834 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:45:06,897 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:45:06,925 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:45:06,925 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:45:06,926 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:45:06,944 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:45:06,947 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:45:06,948 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:45:07,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/java.io.tmpdir/jetty-localhost-33267-hadoop-hdfs-3_4_1-tests_jar-_-any-17802872508826935325/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:45:07,140 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:33267} 2024-11-13T05:45:07,140 INFO [Time-limited test {}] server.Server(415): Started @2819ms 2024-11-13T05:45:07,628 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:45:07,636 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:45:07,638 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:45:07,638 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:45:07,638 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:45:07,639 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:45:07,639 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:45:07,752 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/java.io.tmpdir/jetty-localhost-34839-hadoop-hdfs-3_4_1-tests_jar-_-any-806733499159215998/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:45:07,753 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:34839} 2024-11-13T05:45:07,753 INFO [Time-limited test {}] server.Server(415): Started @3432ms 2024-11-13T05:45:07,804 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:45:07,923 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:45:07,931 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:45:07,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:45:07,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:45:07,936 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:45:07,937 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:45:07,938 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:45:08,043 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/java.io.tmpdir/jetty-localhost-38145-hadoop-hdfs-3_4_1-tests_jar-_-any-2006948378743520397/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:45:08,044 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:38145} 2024-11-13T05:45:08,044 INFO [Time-limited test {}] server.Server(415): Started @3723ms 2024-11-13T05:45:08,046 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:45:08,083 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:45:08,087 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:45:08,091 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:45:08,092 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:45:08,092 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:45:08,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:45:08,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:45:08,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/java.io.tmpdir/jetty-localhost-45771-hadoop-hdfs-3_4_1-tests_jar-_-any-10098211781529438749/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:45:08,202 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:45771} 2024-11-13T05:45:08,202 INFO [Time-limited test {}] server.Server(415): Started @3881ms 2024-11-13T05:45:08,204 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:45:08,985 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data1/current/BP-428696096-172.17.0.2-1731476706240/current, will proceed with Du for space computation calculation, 2024-11-13T05:45:08,985 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data4/current/BP-428696096-172.17.0.2-1731476706240/current, will proceed with Du for space computation calculation, 2024-11-13T05:45:08,985 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data3/current/BP-428696096-172.17.0.2-1731476706240/current, will proceed with Du for space computation calculation, 2024-11-13T05:45:08,985 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data2/current/BP-428696096-172.17.0.2-1731476706240/current, will proceed with Du for space computation calculation, 2024-11-13T05:45:09,026 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:45:09,026 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:45:09,039 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data5/current/BP-428696096-172.17.0.2-1731476706240/current, will proceed with Du for space computation calculation, 2024-11-13T05:45:09,039 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data6/current/BP-428696096-172.17.0.2-1731476706240/current, will proceed with Du for space computation calculation, 2024-11-13T05:45:09,059 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:45:09,083 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x98f6d393b53b767d with lease ID 0x4bfcdb3ffe5ca083: Processing first storage report for DS-a5bab832-1486-42a2-b8c2-317310a1a45d from datanode DatanodeRegistration(127.0.0.1:46375, datanodeUuid=c7c96584-0d7b-4c3e-b65d-ead4dc9341ac, infoPort=42673, infoSecurePort=0, ipcPort=38021, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240) 2024-11-13T05:45:09,084 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x98f6d393b53b767d with lease ID 0x4bfcdb3ffe5ca083: from storage DS-a5bab832-1486-42a2-b8c2-317310a1a45d node DatanodeRegistration(127.0.0.1:46375, datanodeUuid=c7c96584-0d7b-4c3e-b65d-ead4dc9341ac, infoPort=42673, infoSecurePort=0, ipcPort=38021, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:45:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b19a61ffc26a166 with lease ID 0x4bfcdb3ffe5ca084: Processing first storage report for DS-f9c6cef8-ddb9-4773-a618-d0e954165336 from datanode DatanodeRegistration(127.0.0.1:41687, datanodeUuid=b2f5bc00-560f-4425-b733-300d07a1a832, infoPort=41547, infoSecurePort=0, ipcPort=35543, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240) 2024-11-13T05:45:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b19a61ffc26a166 with lease ID 0x4bfcdb3ffe5ca084: from storage DS-f9c6cef8-ddb9-4773-a618-d0e954165336 node DatanodeRegistration(127.0.0.1:41687, datanodeUuid=b2f5bc00-560f-4425-b733-300d07a1a832, infoPort=41547, infoSecurePort=0, ipcPort=35543, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:45:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedd5baffb2152f79 with lease ID 0x4bfcdb3ffe5ca082: Processing first storage report for DS-01408029-007f-41cd-ad85-d56957288564 from datanode DatanodeRegistration(127.0.0.1:42769, datanodeUuid=f85652d7-75af-40d2-be7b-e1cff6d44d06, infoPort=43503, infoSecurePort=0, ipcPort=35781, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240) 2024-11-13T05:45:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedd5baffb2152f79 with lease ID 0x4bfcdb3ffe5ca082: from storage DS-01408029-007f-41cd-ad85-d56957288564 node DatanodeRegistration(127.0.0.1:42769, datanodeUuid=f85652d7-75af-40d2-be7b-e1cff6d44d06, infoPort=43503, infoSecurePort=0, ipcPort=35781, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:45:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x98f6d393b53b767d with lease ID 0x4bfcdb3ffe5ca083: Processing first storage report for DS-99edbf3d-d590-4613-8e95-925f46a79242 from datanode DatanodeRegistration(127.0.0.1:46375, datanodeUuid=c7c96584-0d7b-4c3e-b65d-ead4dc9341ac, infoPort=42673, infoSecurePort=0, ipcPort=38021, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240) 2024-11-13T05:45:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x98f6d393b53b767d with lease ID 0x4bfcdb3ffe5ca083: from storage DS-99edbf3d-d590-4613-8e95-925f46a79242 node DatanodeRegistration(127.0.0.1:46375, datanodeUuid=c7c96584-0d7b-4c3e-b65d-ead4dc9341ac, infoPort=42673, infoSecurePort=0, ipcPort=38021, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:45:09,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b19a61ffc26a166 with lease ID 0x4bfcdb3ffe5ca084: Processing first storage report for DS-091a018b-61d4-4cfa-8bf6-2d31b548bbed from datanode DatanodeRegistration(127.0.0.1:41687, datanodeUuid=b2f5bc00-560f-4425-b733-300d07a1a832, infoPort=41547, infoSecurePort=0, ipcPort=35543, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240) 2024-11-13T05:45:09,086 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b19a61ffc26a166 with lease ID 0x4bfcdb3ffe5ca084: from storage DS-091a018b-61d4-4cfa-8bf6-2d31b548bbed node DatanodeRegistration(127.0.0.1:41687, datanodeUuid=b2f5bc00-560f-4425-b733-300d07a1a832, infoPort=41547, infoSecurePort=0, ipcPort=35543, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:45:09,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xedd5baffb2152f79 with lease ID 0x4bfcdb3ffe5ca082: Processing first storage report for DS-21ad1398-84c3-4f88-840f-35b2411d5939 from datanode DatanodeRegistration(127.0.0.1:42769, datanodeUuid=f85652d7-75af-40d2-be7b-e1cff6d44d06, infoPort=43503, infoSecurePort=0, ipcPort=35781, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240) 2024-11-13T05:45:09,087 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xedd5baffb2152f79 with lease ID 0x4bfcdb3ffe5ca082: from storage DS-21ad1398-84c3-4f88-840f-35b2411d5939 node DatanodeRegistration(127.0.0.1:42769, datanodeUuid=f85652d7-75af-40d2-be7b-e1cff6d44d06, infoPort=43503, infoSecurePort=0, ipcPort=35781, storageInfo=lv=-57;cid=testClusterID;nsid=4529235;c=1731476706240), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:45:09,110 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a 2024-11-13T05:45:09,177 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/zookeeper_0, clientPort=60627, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:45:09,195 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60627 2024-11-13T05:45:09,211 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:09,215 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:09,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:45:09,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:45:09,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:45:09,810 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d with version=8 2024-11-13T05:45:09,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/hbase-staging 2024-11-13T05:45:10,093 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:45:10,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,105 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:45:10,106 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,106 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:45:10,236 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:45:10,289 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T05:45:10,297 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T05:45:10,301 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:45:10,324 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21771 (auto-detected) 2024-11-13T05:45:10,325 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T05:45:10,343 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38153 2024-11-13T05:45:10,363 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38153 connecting to ZooKeeper ensemble=127.0.0.1:60627 2024-11-13T05:45:10,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:381530x0, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:45:10,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38153-0x101340371ed0000 connected 2024-11-13T05:45:10,571 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,592 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:45:10,597 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d, hbase.cluster.distributed=false 2024-11-13T05:45:10,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:45:10,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38153 2024-11-13T05:45:10,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38153 2024-11-13T05:45:10,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38153 2024-11-13T05:45:10,627 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38153 2024-11-13T05:45:10,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38153 2024-11-13T05:45:10,727 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:45:10,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,729 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:45:10,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,729 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:45:10,732 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:45:10,735 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:45:10,736 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45573 2024-11-13T05:45:10,738 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45573 connecting to ZooKeeper ensemble=127.0.0.1:60627 2024-11-13T05:45:10,739 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,744 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,787 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455730x0, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:45:10,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:455730x0, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:45:10,788 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45573-0x101340371ed0001 connected 2024-11-13T05:45:10,791 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:45:10,799 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:45:10,801 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:45:10,806 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:45:10,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45573 2024-11-13T05:45:10,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45573 2024-11-13T05:45:10,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45573 2024-11-13T05:45:10,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45573 2024-11-13T05:45:10,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45573 2024-11-13T05:45:10,828 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:45:10,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,829 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:45:10,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:45:10,829 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:45:10,829 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:45:10,830 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46279 2024-11-13T05:45:10,832 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46279 connecting to ZooKeeper ensemble=127.0.0.1:60627 2024-11-13T05:45:10,833 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,836 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462790x0, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:45:10,855 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:462790x0, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:45:10,855 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46279-0x101340371ed0002 connected 2024-11-13T05:45:10,856 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:45:10,857 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:45:10,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:45:10,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:45:10,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46279 2024-11-13T05:45:10,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46279 2024-11-13T05:45:10,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46279 2024-11-13T05:45:10,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46279 2024-11-13T05:45:10,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46279 2024-11-13T05:45:10,884 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:45:10,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,884 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:45:10,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:45:10,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:45:10,885 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:45:10,885 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:45:10,886 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34269 2024-11-13T05:45:10,888 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34269 connecting to ZooKeeper ensemble=127.0.0.1:60627 2024-11-13T05:45:10,889 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,892 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:10,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342690x0, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:45:10,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:342690x0, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:45:10,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34269-0x101340371ed0003 connected 2024-11-13T05:45:10,905 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:45:10,906 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:45:10,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:45:10,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:45:10,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34269 2024-11-13T05:45:10,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34269 2024-11-13T05:45:10,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34269 2024-11-13T05:45:10,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34269 2024-11-13T05:45:10,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34269 2024-11-13T05:45:10,927 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bc386b7bf2fe:38153 2024-11-13T05:45:10,928 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:10,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,946 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,948 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:10,983 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:45:10,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:45:10,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:10,984 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:10,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:10,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:45:10,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:10,985 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:45:10,986 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bc386b7bf2fe,38153,1731476709944 from backup master directory 2024-11-13T05:45:10,995 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:10,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:45:10,996 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:45:10,997 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:10,999 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T05:45:11,000 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T05:45:11,057 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/hbase.id] with ID: ca14ade9-6f32-4196-b83a-8a2c34a32d4c 2024-11-13T05:45:11,057 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/.tmp/hbase.id 2024-11-13T05:45:11,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:45:11,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:45:11,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:45:11,072 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/.tmp/hbase.id]:[hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/hbase.id] 2024-11-13T05:45:11,124 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:45:11,130 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:45:11,150 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-13T05:45:11,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,170 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:45:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:45:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:45:11,209 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:45:11,211 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:45:11,217 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:11,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:45:11,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:45:11,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:45:11,266 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store 2024-11-13T05:45:11,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:45:11,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:45:11,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:45:11,295 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T05:45:11,299 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:11,300 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:45:11,301 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:11,301 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:11,303 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:45:11,303 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:11,304 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:11,305 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476711300Disabling compacts and flushes for region at 1731476711300Disabling writes for close at 1731476711303 (+3 ms)Writing region close event to WAL at 1731476711304 (+1 ms)Closed at 1731476711304 2024-11-13T05:45:11,307 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/.initializing 2024-11-13T05:45:11,308 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/WALs/bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:11,316 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:11,332 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C38153%2C1731476709944, suffix=, logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/WALs/bc386b7bf2fe,38153,1731476709944, archiveDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/oldWALs, maxLogs=10 2024-11-13T05:45:11,360 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/WALs/bc386b7bf2fe,38153,1731476709944/bc386b7bf2fe%2C38153%2C1731476709944.1731476711336, exclude list is [], retry=0 2024-11-13T05:45:11,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:11,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:11,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:11,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:11,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T05:45:11,415 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-13T05:45:11,472 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/WALs/bc386b7bf2fe,38153,1731476709944/bc386b7bf2fe%2C38153%2C1731476709944.1731476711336 2024-11-13T05:45:11,473 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:11,474 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:11,474 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:11,477 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,477 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:45:11,537 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:11,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:11,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:45:11,543 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:11,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:11,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:45:11,548 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:11,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:11,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,552 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:45:11,552 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:11,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:11,554 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,557 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,558 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,564 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,564 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,567 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:45:11,571 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:45:11,575 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:11,576 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61089994, jitterRate=-0.08968815207481384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:45:11,584 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731476711488Initializing all the Stores at 1731476711491 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476711491Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476711492 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476711492Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476711492Cleaning up temporary data from old regions at 1731476711564 (+72 ms)Region opened successfully at 1731476711584 (+20 ms) 2024-11-13T05:45:11,586 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:45:11,616 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519bfc39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:45:11,643 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:45:11,654 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:45:11,654 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:45:11,656 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:45:11,658 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:45:11,662 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T05:45:11,662 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:45:11,686 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:45:11,694 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:45:11,745 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:45:11,750 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:45:11,754 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:45:11,761 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:45:11,763 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:45:11,766 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:45:11,775 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:45:11,777 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:45:11,786 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:45:11,808 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:45:11,819 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:45:11,828 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:45:11,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:45:11,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:45:11,828 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:45:11,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,833 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bc386b7bf2fe,38153,1731476709944, sessionid=0x101340371ed0000, setting cluster-up flag (Was=false) 2024-11-13T05:45:11,861 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,887 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:45:11,891 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:11,912 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:11,936 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:45:11,939 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:11,947 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:45:12,012 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:45:12,016 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(746): ClusterId : ca14ade9-6f32-4196-b83a-8a2c34a32d4c 2024-11-13T05:45:12,016 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(746): ClusterId : ca14ade9-6f32-4196-b83a-8a2c34a32d4c 2024-11-13T05:45:12,016 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(746): ClusterId : ca14ade9-6f32-4196-b83a-8a2c34a32d4c 2024-11-13T05:45:12,019 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:45:12,019 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:45:12,019 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:45:12,022 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:45:12,028 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:45:12,033 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bc386b7bf2fe,38153,1731476709944 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:45:12,046 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:45:12,046 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:45:12,046 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:45:12,047 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:45:12,047 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:45:12,047 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:45:12,047 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:45:12,047 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:45:12,048 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:45:12,048 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:45:12,048 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bc386b7bf2fe:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:45:12,048 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,048 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:45:12,048 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,049 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731476742049 2024-11-13T05:45:12,051 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:45:12,052 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:45:12,054 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:45:12,055 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:45:12,056 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:45:12,057 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:45:12,057 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:45:12,058 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:45:12,059 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,062 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,063 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:45:12,063 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:45:12,064 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:45:12,064 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:45:12,066 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:45:12,067 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:45:12,068 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:45:12,068 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:45:12,068 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:45:12,068 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731476712068,5,FailOnTimeoutGroup] 2024-11-13T05:45:12,069 DEBUG [RS:0;bc386b7bf2fe:45573 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d93fb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:45:12,069 DEBUG [RS:2;bc386b7bf2fe:34269 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8e243c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:45:12,069 DEBUG [RS:1;bc386b7bf2fe:46279 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd1714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:45:12,069 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731476712069,5,FailOnTimeoutGroup] 2024-11-13T05:45:12,069 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,069 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:45:12,070 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,071 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,107 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;bc386b7bf2fe:34269 2024-11-13T05:45:12,107 DEBUG [RS:1;bc386b7bf2fe:46279 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bc386b7bf2fe:46279 2024-11-13T05:45:12,107 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bc386b7bf2fe:45573 2024-11-13T05:45:12,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:45:12,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:45:12,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:45:12,111 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:45:12,111 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:45:12,111 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:45:12,112 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:45:12,112 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:45:12,112 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:45:12,112 DEBUG [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:45:12,112 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:45:12,112 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:45:12,112 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:45:12,113 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d 2024-11-13T05:45:12,115 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,38153,1731476709944 with port=45573, startcode=1731476710694 2024-11-13T05:45:12,115 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,38153,1731476709944 with port=34269, startcode=1731476710883 2024-11-13T05:45:12,115 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,38153,1731476709944 with port=46279, startcode=1731476710827 2024-11-13T05:45:12,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:45:12,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:45:12,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:45:12,129 DEBUG [RS:0;bc386b7bf2fe:45573 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:45:12,129 DEBUG [RS:1;bc386b7bf2fe:46279 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:45:12,129 DEBUG [RS:2;bc386b7bf2fe:34269 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:45:12,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:12,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:45:12,137 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:45:12,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,138 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:45:12,142 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:45:12,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:45:12,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:45:12,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:45:12,152 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:45:12,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:45:12,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740 2024-11-13T05:45:12,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740 2024-11-13T05:45:12,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:45:12,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:45:12,164 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:45:12,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:45:12,169 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43699, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:45:12,169 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:45:12,169 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42701, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:45:12,172 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:12,173 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63722277, jitterRate=-0.05046407878398895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:45:12,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38153 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:12,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731476712131Initializing all the Stores at 1731476712133 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476712133Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476712133Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476712133Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476712133Cleaning up temporary data from old regions at 1731476712163 (+30 ms)Region opened successfully at 1731476712178 (+15 ms) 2024-11-13T05:45:12,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:45:12,179 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:45:12,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:45:12,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:45:12,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:45:12,179 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38153 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:12,181 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:45:12,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476712179Disabling compacts and flushes for region at 1731476712179Disabling writes for close at 1731476712179Writing region close event to WAL at 1731476712180 (+1 ms)Closed at 1731476712181 (+1 ms) 2024-11-13T05:45:12,185 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:45:12,185 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:45:12,193 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38153 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:12,193 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38153 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:12,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:45:12,197 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38153 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:12,198 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38153 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:12,199 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d 2024-11-13T05:45:12,199 DEBUG [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d 2024-11-13T05:45:12,199 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39659 2024-11-13T05:45:12,199 DEBUG [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39659 2024-11-13T05:45:12,200 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:45:12,200 DEBUG [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:45:12,202 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d 2024-11-13T05:45:12,202 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39659 2024-11-13T05:45:12,203 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:45:12,206 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:45:12,211 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:45:12,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:45:12,252 DEBUG [RS:1;bc386b7bf2fe:46279 {}] zookeeper.ZKUtil(111): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:12,252 DEBUG [RS:0;bc386b7bf2fe:45573 {}] zookeeper.ZKUtil(111): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:12,252 WARN [RS:1;bc386b7bf2fe:46279 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:45:12,252 WARN [RS:0;bc386b7bf2fe:45573 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:45:12,252 DEBUG [RS:2;bc386b7bf2fe:34269 {}] zookeeper.ZKUtil(111): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:12,252 INFO [RS:1;bc386b7bf2fe:46279 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:12,252 WARN [RS:2;bc386b7bf2fe:34269 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:45:12,252 INFO [RS:0;bc386b7bf2fe:45573 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:12,253 INFO [RS:2;bc386b7bf2fe:34269 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:12,253 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:12,253 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:12,253 DEBUG [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:12,254 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,45573,1731476710694] 2024-11-13T05:45:12,254 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,34269,1731476710883] 2024-11-13T05:45:12,254 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,46279,1731476710827] 2024-11-13T05:45:12,280 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:45:12,280 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:45:12,280 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:45:12,297 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:45:12,297 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:45:12,297 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:45:12,303 INFO [RS:1;bc386b7bf2fe:46279 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:45:12,303 INFO [RS:2;bc386b7bf2fe:34269 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:45:12,303 INFO [RS:0;bc386b7bf2fe:45573 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:45:12,303 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,303 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,303 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,304 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:45:12,304 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:45:12,305 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:45:12,312 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:45:12,312 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:45:12,314 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,314 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,314 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,314 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,314 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:45:12,315 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:45:12,315 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,315 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,316 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:45:12,317 DEBUG [RS:1;bc386b7bf2fe:46279 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:45:12,317 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:45:12,317 DEBUG [RS:2;bc386b7bf2fe:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:45:12,319 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:45:12,320 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,320 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,320 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,320 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,320 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,320 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,320 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:45:12,321 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,321 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,321 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,321 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,321 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,321 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:45:12,321 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:45:12,322 DEBUG [RS:0;bc386b7bf2fe:45573 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:45:12,325 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,34269,1731476710883-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,326 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,327 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,327 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,327 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,46279,1731476710827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:45:12,328 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,329 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,329 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,329 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,329 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,329 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45573,1731476710694-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:45:12,348 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:45:12,349 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:45:12,350 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45573,1731476710694-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,350 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,34269,1731476710883-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,350 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,351 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,351 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.Replication(171): bc386b7bf2fe,34269,1731476710883 started 2024-11-13T05:45:12,351 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.Replication(171): bc386b7bf2fe,45573,1731476710694 started 2024-11-13T05:45:12,351 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:45:12,352 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,46279,1731476710827-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,352 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,352 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.Replication(171): bc386b7bf2fe,46279,1731476710827 started 2024-11-13T05:45:12,362 WARN [bc386b7bf2fe:38153 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:45:12,371 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,371 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,371 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,45573,1731476710694, RpcServer on bc386b7bf2fe/172.17.0.2:45573, sessionid=0x101340371ed0001 2024-11-13T05:45:12,371 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,34269,1731476710883, RpcServer on bc386b7bf2fe/172.17.0.2:34269, sessionid=0x101340371ed0003 2024-11-13T05:45:12,372 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:45:12,372 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:45:12,372 DEBUG [RS:2;bc386b7bf2fe:34269 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:12,372 DEBUG [RS:0;bc386b7bf2fe:45573 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:12,372 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,34269,1731476710883' 2024-11-13T05:45:12,372 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45573,1731476710694' 2024-11-13T05:45:12,372 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:45:12,372 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:45:12,373 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:45:12,373 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:45:12,374 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:45:12,374 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:45:12,374 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:12,374 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:45:12,374 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:45:12,374 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,46279,1731476710827, RpcServer on bc386b7bf2fe/172.17.0.2:46279, sessionid=0x101340371ed0002 2024-11-13T05:45:12,374 DEBUG [RS:0;bc386b7bf2fe:45573 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:12,374 DEBUG [RS:2;bc386b7bf2fe:34269 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:12,374 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45573,1731476710694' 2024-11-13T05:45:12,374 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,34269,1731476710883' 2024-11-13T05:45:12,374 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:45:12,374 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:45:12,374 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:45:12,375 DEBUG [RS:1;bc386b7bf2fe:46279 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:12,375 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,46279,1731476710827' 2024-11-13T05:45:12,375 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:45:12,375 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:45:12,375 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:45:12,375 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:45:12,375 DEBUG [RS:2;bc386b7bf2fe:34269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:45:12,376 DEBUG [RS:0;bc386b7bf2fe:45573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:45:12,376 INFO [RS:2;bc386b7bf2fe:34269 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:45:12,376 INFO [RS:0;bc386b7bf2fe:45573 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:45:12,376 INFO [RS:2;bc386b7bf2fe:34269 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:45:12,376 INFO [RS:0;bc386b7bf2fe:45573 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:45:12,376 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:45:12,376 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:45:12,376 DEBUG [RS:1;bc386b7bf2fe:46279 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:12,376 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,46279,1731476710827' 2024-11-13T05:45:12,376 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:45:12,376 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:45:12,377 DEBUG [RS:1;bc386b7bf2fe:46279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:45:12,377 INFO [RS:1;bc386b7bf2fe:46279 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:45:12,377 INFO [RS:1;bc386b7bf2fe:46279 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:45:12,480 INFO [RS:2;bc386b7bf2fe:34269 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:12,480 INFO [RS:0;bc386b7bf2fe:45573 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:12,480 INFO [RS:1;bc386b7bf2fe:46279 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:12,484 INFO [RS:0;bc386b7bf2fe:45573 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C45573%2C1731476710694, suffix=, logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,45573,1731476710694, archiveDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs, maxLogs=32 2024-11-13T05:45:12,484 INFO [RS:1;bc386b7bf2fe:46279 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C46279%2C1731476710827, suffix=, logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,46279,1731476710827, archiveDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs, maxLogs=32 2024-11-13T05:45:12,484 INFO [RS:2;bc386b7bf2fe:34269 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C34269%2C1731476710883, suffix=, logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883, archiveDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs, maxLogs=32 2024-11-13T05:45:12,501 DEBUG [RS:1;bc386b7bf2fe:46279 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,46279,1731476710827/bc386b7bf2fe%2C46279%2C1731476710827.1731476712487, exclude list is [], retry=0 2024-11-13T05:45:12,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:12,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:12,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:12,507 DEBUG [RS:0;bc386b7bf2fe:45573 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,45573,1731476710694/bc386b7bf2fe%2C45573%2C1731476710694.1731476712488, exclude list is [], retry=0 2024-11-13T05:45:12,507 DEBUG [RS:2;bc386b7bf2fe:34269 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, exclude list is [], retry=0 2024-11-13T05:45:12,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:12,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:12,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:12,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:12,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:12,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:12,547 INFO [RS:1;bc386b7bf2fe:46279 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,46279,1731476710827/bc386b7bf2fe%2C46279%2C1731476710827.1731476712487 2024-11-13T05:45:12,547 DEBUG [RS:1;bc386b7bf2fe:46279 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:12,559 INFO [RS:2;bc386b7bf2fe:34269 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 2024-11-13T05:45:12,560 DEBUG [RS:2;bc386b7bf2fe:34269 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:12,561 INFO [RS:0;bc386b7bf2fe:45573 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,45573,1731476710694/bc386b7bf2fe%2C45573%2C1731476710694.1731476712488 2024-11-13T05:45:12,561 DEBUG [RS:0;bc386b7bf2fe:45573 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:12,615 DEBUG [bc386b7bf2fe:38153 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T05:45:12,626 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:45:12,633 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:45:12,633 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:45:12,633 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T05:45:12,633 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:45:12,633 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:45:12,633 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T05:45:12,634 INFO [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:45:12,634 INFO [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:45:12,634 INFO [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T05:45:12,634 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:45:12,640 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:12,645 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,34269,1731476710883, state=OPENING 2024-11-13T05:45:12,691 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:45:12,703 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:12,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:12,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:12,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:12,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:12,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:12,705 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:12,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:12,709 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:45:12,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,34269,1731476710883}] 2024-11-13T05:45:12,894 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:45:12,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47721, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:45:12,909 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:45:12,909 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:12,910 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:45:12,913 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C34269%2C1731476710883.meta, suffix=.meta, logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883, archiveDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs, maxLogs=32 2024-11-13T05:45:12,929 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, exclude list is [], retry=0 2024-11-13T05:45:12,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:12,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:12,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:12,963 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta 2024-11-13T05:45:12,964 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:12,964 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:12,966 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:45:12,969 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:45:12,973 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:45:12,978 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:45:12,979 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:12,979 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:45:12,979 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:45:12,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:45:12,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:45:12,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:45:12,989 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:45:12,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:45:12,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:45:12,992 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:45:12,995 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:45:12,995 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:12,996 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:12,997 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:45:12,998 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740 2024-11-13T05:45:13,001 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740 2024-11-13T05:45:13,004 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:45:13,004 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:45:13,005 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:45:13,009 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:45:13,011 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58755980, jitterRate=-0.12446767091751099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:45:13,011 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:45:13,013 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476712980Writing region info on filesystem at 1731476712980Initializing all the Stores at 1731476712982 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476712982Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476712983 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476712983Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476712983Cleaning up temporary data from old regions at 1731476713004 (+21 ms)Running coprocessor post-open hooks at 1731476713011 (+7 ms)Region opened successfully at 1731476713012 (+1 ms) 2024-11-13T05:45:13,020 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731476712886 2024-11-13T05:45:13,032 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:45:13,033 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:45:13,034 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:13,037 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,34269,1731476710883, state=OPEN 2024-11-13T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:13,053 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:13,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:13,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:13,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:13,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:13,054 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:13,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:45:13,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,34269,1731476710883 in 342 msec 2024-11-13T05:45:13,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:45:13,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 871 msec 2024-11-13T05:45:13,071 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:45:13,071 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:45:13,090 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:45:13,092 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1] 2024-11-13T05:45:13,112 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:45:13,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46319, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:45:13,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1570 sec 2024-11-13T05:45:13,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731476713134, completionTime=-1 2024-11-13T05:45:13,138 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T05:45:13,138 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:45:13,182 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-13T05:45:13,182 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731476773182 2024-11-13T05:45:13,183 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731476833183 2024-11-13T05:45:13,183 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 44 msec 2024-11-13T05:45:13,185 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:45:13,198 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38153,1731476709944-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:13,198 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38153,1731476709944-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:13,199 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38153,1731476709944-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:13,201 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bc386b7bf2fe:38153, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:13,201 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:13,202 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:13,207 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:45:13,234 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.237sec 2024-11-13T05:45:13,236 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:45:13,237 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:45:13,238 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:45:13,239 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:45:13,239 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:45:13,240 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38153,1731476709944-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:45:13,240 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38153,1731476709944-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:45:13,245 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:45:13,246 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:45:13,247 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38153,1731476709944-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:13,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed908a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:45:13,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,38153,-1 for getting cluster id 2024-11-13T05:45:13,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:45:13,343 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ca14ade9-6f32-4196-b83a-8a2c34a32d4c' 2024-11-13T05:45:13,346 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:45:13,346 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ca14ade9-6f32-4196-b83a-8a2c34a32d4c" 2024-11-13T05:45:13,346 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bd7bd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:45:13,346 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,38153,-1] 2024-11-13T05:45:13,348 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:45:13,350 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:13,351 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:45:13,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769aece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:45:13,354 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:45:13,363 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1] 2024-11-13T05:45:13,363 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:45:13,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:45:13,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:13,388 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:39659/hbase 2024-11-13T05:45:13,401 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=7843 2024-11-13T05:45:13,420 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:13,423 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:13,424 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:13,429 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-76123985, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-76123985, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:13,443 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-76123985/hregion-76123985.1731476713431, exclude list is [], retry=0 2024-11-13T05:45:13,448 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:13,448 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:13,449 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:13,478 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-76123985/hregion-76123985.1731476713431 2024-11-13T05:45:13,479 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:13,479 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ee6fb674c1a7bd6a8b3e2bf1be7af80a, NAME => 'testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:13,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741838_1014 (size=64) 2024-11-13T05:45:13,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741838_1014 (size=64) 2024-11-13T05:45:13,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741838_1014 (size=64) 2024-11-13T05:45:13,493 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:13,495 INFO [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,497 INFO [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6fb674c1a7bd6a8b3e2bf1be7af80a columnFamilyName a 2024-11-13T05:45:13,498 DEBUG [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:13,499 INFO [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] regionserver.HStore(327): Store=ee6fb674c1a7bd6a8b3e2bf1be7af80a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:13,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,502 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,502 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,507 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,511 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:13,512 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee6fb674c1a7bd6a8b3e2bf1be7af80a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66077697, jitterRate=-0.015365585684776306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:13,513 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee6fb674c1a7bd6a8b3e2bf1be7af80a: Writing region info on filesystem at 1731476713493Initializing all the Stores at 1731476713494 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476713494Cleaning up temporary data from old regions at 1731476713502 (+8 ms)Region opened successfully at 1731476713513 (+11 ms) 2024-11-13T05:45:13,513 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee6fb674c1a7bd6a8b3e2bf1be7af80a, disabling compactions & flushes 2024-11-13T05:45:13,513 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a. 2024-11-13T05:45:13,513 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a. 2024-11-13T05:45:13,513 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a. after waiting 0 ms 2024-11-13T05:45:13,513 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a. 2024-11-13T05:45:13,514 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a. 2024-11-13T05:45:13,514 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee6fb674c1a7bd6a8b3e2bf1be7af80a: Waiting for close lock at 1731476713513Disabling compacts and flushes for region at 1731476713513Disabling writes for close at 1731476713513Writing region close event to WAL at 1731476713514 (+1 ms)Closed at 1731476713514 2024-11-13T05:45:13,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741837_1013 (size=175) 2024-11-13T05:45:13,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741837_1013 (size=175) 2024-11-13T05:45:13,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741837_1013 (size=175) 2024-11-13T05:45:13,527 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:13,527 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-76123985:(num 1731476713431) 2024-11-13T05:45:13,529 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T05:45:13,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741839_1015 (size=469) 2024-11-13T05:45:13,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741839_1015 (size=469) 2024-11-13T05:45:13,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741839_1015 (size=469) 2024-11-13T05:45:13,571 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T05:45:13,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741840_1016 (size=322) 2024-11-13T05:45:13,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741840_1016 (size=322) 2024-11-13T05:45:13,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741840_1016 (size=322) 2024-11-13T05:45:13,630 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1, size=469 (469bytes) 2024-11-13T05:45:13,630 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T05:45:13,630 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T05:45:13,631 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1 2024-11-13T05:45:13,635 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1 after 2ms 2024-11-13T05:45:13,665 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:13,665 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1 took 37ms 2024-11-13T05:45:13,670 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1 so closing down 2024-11-13T05:45:13,670 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:13,673 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T05:45:13,698 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T05:45:13,699 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:13,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741841_1017 (size=469) 2024-11-13T05:45:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741841_1017 (size=469) 2024-11-13T05:45:13,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741841_1017 (size=469) 2024-11-13T05:45:13,711 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:13,714 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002 2024-11-13T05:45:13,718 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 48 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T05:45:13,718 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1, journal: Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1, size=469 (469bytes) at 1731476713630Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1 so closing down at 1731476713670 (+40 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000001-wal-1.temp at 1731476713698 (+28 ms)3 split writer threads finished at 1731476713699 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731476713711 (+12 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002 at 1731476713714 (+3 ms)Processed 2 edits across 1 Regions in 48 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731476713718 (+4 ms) 2024-11-13T05:45:13,732 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2, size=322 (322bytes) 2024-11-13T05:45:13,732 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2 2024-11-13T05:45:13,733 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2 after 0ms 2024-11-13T05:45:13,760 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:13,760 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2 took 28ms 2024-11-13T05:45:13,763 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2 so closing down 2024-11-13T05:45:13,763 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:13,765 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T05:45:13,789 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T05:45:13,790 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741842_1018 (size=322) 2024-11-13T05:45:13,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741842_1018 (size=322) 2024-11-13T05:45:13,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741842_1018 (size=322) 2024-11-13T05:45:13,799 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:13,829 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:13,831 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-13T05:45:13,833 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T05:45:13,833 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2, journal: Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2, size=322 (322bytes) at 1731476713732Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2 so closing down at 1731476713763 (+31 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002-wal-2.temp at 1731476713789 (+26 ms)3 split writer threads finished at 1731476713790 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731476713799 (+9 ms)Processed 1 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731476713833 (+34 ms) 2024-11-13T05:45:13,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:13,836 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:13,850 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal.1731476713836, exclude list is [], retry=0 2024-11-13T05:45:13,854 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:13,854 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:13,855 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:13,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal.1731476713836 2024-11-13T05:45:13,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:13,883 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee6fb674c1a7bd6a8b3e2bf1be7af80a, NAME => 'testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:13,883 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:13,883 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,883 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,885 INFO [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,887 INFO [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee6fb674c1a7bd6a8b3e2bf1be7af80a columnFamilyName a 2024-11-13T05:45:13,887 DEBUG [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:13,888 INFO [StoreOpener-ee6fb674c1a7bd6a8b3e2bf1be7af80a-1 {}] regionserver.HStore(327): Store=ee6fb674c1a7bd6a8b3e2bf1be7af80a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:13,888 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,889 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:13,894 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002 2024-11-13T05:45:13,921 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:13,926 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002 2024-11-13T05:45:13,930 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee6fb674c1a7bd6a8b3e2bf1be7af80a 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T05:45:13,976 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/.tmp/a/ea567b3992f8434997a88bfde9e56262 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731476713527/Put/seqid=0 2024-11-13T05:45:13,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741844_1020 (size=5170) 2024-11-13T05:45:13,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741844_1020 (size=5170) 2024-11-13T05:45:13,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741844_1020 (size=5170) 2024-11-13T05:45:13,988 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/.tmp/a/ea567b3992f8434997a88bfde9e56262 2024-11-13T05:45:14,031 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/.tmp/a/ea567b3992f8434997a88bfde9e56262 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/a/ea567b3992f8434997a88bfde9e56262 2024-11-13T05:45:14,041 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/a/ea567b3992f8434997a88bfde9e56262, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T05:45:14,046 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ee6fb674c1a7bd6a8b3e2bf1be7af80a in 115ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T05:45:14,047 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/0000000000000000002 2024-11-13T05:45:14,048 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:14,048 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:14,051 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee6fb674c1a7bd6a8b3e2bf1be7af80a 2024-11-13T05:45:14,054 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/ee6fb674c1a7bd6a8b3e2bf1be7af80a/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T05:45:14,056 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee6fb674c1a7bd6a8b3e2bf1be7af80a; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63622219, jitterRate=-0.05195505917072296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:14,057 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee6fb674c1a7bd6a8b3e2bf1be7af80a: Writing region info on filesystem at 1731476713884Initializing all the Stores at 1731476713885 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476713885Obtaining lock to block concurrent updates at 1731476713930 (+45 ms)Preparing flush snapshotting stores in ee6fb674c1a7bd6a8b3e2bf1be7af80a at 1731476713930Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731476713933 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731476713421.ee6fb674c1a7bd6a8b3e2bf1be7af80a. at 1731476713933Flushing ee6fb674c1a7bd6a8b3e2bf1be7af80a/a: creating writer at 1731476713935 (+2 ms)Flushing ee6fb674c1a7bd6a8b3e2bf1be7af80a/a: appending metadata at 1731476713966 (+31 ms)Flushing ee6fb674c1a7bd6a8b3e2bf1be7af80a/a: closing flushed file at 1731476713968 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15ac799a: reopening flushed file at 1731476714029 (+61 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ee6fb674c1a7bd6a8b3e2bf1be7af80a in 115ms, sequenceid=2, compaction requested=false; wal=null at 1731476714046 (+17 ms)Cleaning up temporary data from old regions at 1731476714048 (+2 ms)Region opened successfully at 1731476714057 (+9 ms) 2024-11-13T05:45:14,079 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=373 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:38536 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:39659/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:39324 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:56870 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:53138 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:50752 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:44216 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=7830 (was 7843) 2024-11-13T05:45:14,089 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=7830 2024-11-13T05:45:14,106 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:14,108 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:14,109 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:14,112 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73496381, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-73496381, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:14,127 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73496381/hregion-73496381.1731476714113, exclude list is [], retry=0 2024-11-13T05:45:14,131 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:14,131 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:14,132 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:14,157 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73496381/hregion-73496381.1731476714113 2024-11-13T05:45:14,157 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:14,157 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b4ae320d394710486e2563f4c5d712c4, NAME => 'testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741846_1022 (size=64) 2024-11-13T05:45:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741846_1022 (size=64) 2024-11-13T05:45:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741846_1022 (size=64) 2024-11-13T05:45:14,171 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:14,173 INFO [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,175 INFO [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ae320d394710486e2563f4c5d712c4 columnFamilyName a 2024-11-13T05:45:14,176 DEBUG [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:14,176 INFO [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] regionserver.HStore(327): Store=b4ae320d394710486e2563f4c5d712c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:14,176 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,177 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,178 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,178 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,178 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,181 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,184 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:14,185 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4ae320d394710486e2563f4c5d712c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74362881, jitterRate=0.1080932766199112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:14,185 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4ae320d394710486e2563f4c5d712c4: Writing region info on filesystem at 1731476714171Initializing all the Stores at 1731476714172 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476714172Cleaning up temporary data from old regions at 1731476714178 (+6 ms)Region opened successfully at 1731476714185 (+7 ms) 2024-11-13T05:45:14,185 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b4ae320d394710486e2563f4c5d712c4, disabling compactions & flushes 2024-11-13T05:45:14,185 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4. 2024-11-13T05:45:14,185 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4. 2024-11-13T05:45:14,185 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4. after waiting 0 ms 2024-11-13T05:45:14,185 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4. 2024-11-13T05:45:14,186 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4. 2024-11-13T05:45:14,186 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b4ae320d394710486e2563f4c5d712c4: Waiting for close lock at 1731476714185Disabling compacts and flushes for region at 1731476714185Disabling writes for close at 1731476714185Writing region close event to WAL at 1731476714186 (+1 ms)Closed at 1731476714186 2024-11-13T05:45:14,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741845_1021 (size=175) 2024-11-13T05:45:14,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741845_1021 (size=175) 2024-11-13T05:45:14,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741845_1021 (size=175) 2024-11-13T05:45:14,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:14,193 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73496381:(num 1731476714113) 2024-11-13T05:45:14,194 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T05:45:14,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741847_1023 (size=469) 2024-11-13T05:45:14,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741847_1023 (size=469) 2024-11-13T05:45:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741847_1023 (size=469) 2024-11-13T05:45:14,238 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T05:45:14,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741848_1024 (size=322) 2024-11-13T05:45:14,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741848_1024 (size=322) 2024-11-13T05:45:14,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741848_1024 (size=322) 2024-11-13T05:45:14,288 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2, size=322 (322bytes) 2024-11-13T05:45:14,289 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2 2024-11-13T05:45:14,290 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2 after 0ms 2024-11-13T05:45:14,316 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:14,316 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2 took 28ms 2024-11-13T05:45:14,318 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2 so closing down 2024-11-13T05:45:14,318 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:14,321 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T05:45:14,349 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T05:45:14,349 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:14,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741849_1025 (size=322) 2024-11-13T05:45:14,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741849_1025 (size=322) 2024-11-13T05:45:14,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741849_1025 (size=322) 2024-11-13T05:45:14,358 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:14,360 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002 2024-11-13T05:45:14,360 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T05:45:14,361 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2, journal: Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2, size=322 (322bytes) at 1731476714289Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2 so closing down at 1731476714318 (+29 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002-wal-2.temp at 1731476714349 (+31 ms)3 split writer threads finished at 1731476714349Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731476714358 (+9 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002 at 1731476714360 (+2 ms)Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731476714361 (+1 ms) 2024-11-13T05:45:14,377 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1, size=469 (469bytes) 2024-11-13T05:45:14,377 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1 2024-11-13T05:45:14,377 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1 after 0ms 2024-11-13T05:45:14,403 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:14,403 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1 took 27ms 2024-11-13T05:45:14,406 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1 so closing down 2024-11-13T05:45:14,407 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:14,409 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T05:45:14,435 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T05:45:14,436 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:14,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741850_1026 (size=469) 2024-11-13T05:45:14,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741850_1026 (size=469) 2024-11-13T05:45:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741850_1026 (size=469) 2024-11-13T05:45:14,445 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:14,473 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:14,476 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002, length=322 2024-11-13T05:45:14,478 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002 2024-11-13T05:45:14,478 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T05:45:14,479 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1, journal: Splitting hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1, size=469 (469bytes) at 1731476714377Finishing writing output for hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1 so closing down at 1731476714406 (+29 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000001-wal-1.temp at 1731476714435 (+29 ms)3 split writer threads finished at 1731476714436 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731476714445 (+9 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002 at 1731476714478 (+33 ms)Processed 2 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731476714479 (+1 ms) 2024-11-13T05:45:14,479 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:14,481 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:14,494 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal.1731476714482, exclude list is [], retry=0 2024-11-13T05:45:14,498 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:14,499 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:14,499 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:14,526 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal.1731476714482 2024-11-13T05:45:14,526 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:14,527 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b4ae320d394710486e2563f4c5d712c4, NAME => 'testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:14,527 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:14,527 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,527 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,529 INFO [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,531 INFO [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4ae320d394710486e2563f4c5d712c4 columnFamilyName a 2024-11-13T05:45:14,532 DEBUG [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:14,532 INFO [StoreOpener-b4ae320d394710486e2563f4c5d712c4-1 {}] regionserver.HStore(327): Store=b4ae320d394710486e2563f4c5d712c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:14,533 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,536 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,536 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002 2024-11-13T05:45:14,564 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:14,565 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002 2024-11-13T05:45:14,565 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b4ae320d394710486e2563f4c5d712c4 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T05:45:14,580 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/.tmp/a/b850860b17e7469e8050be37f6e06cda is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731476714193/Put/seqid=0 2024-11-13T05:45:14,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741852_1028 (size=5170) 2024-11-13T05:45:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741852_1028 (size=5170) 2024-11-13T05:45:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741852_1028 (size=5170) 2024-11-13T05:45:14,590 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/.tmp/a/b850860b17e7469e8050be37f6e06cda 2024-11-13T05:45:14,599 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/.tmp/a/b850860b17e7469e8050be37f6e06cda as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/a/b850860b17e7469e8050be37f6e06cda 2024-11-13T05:45:14,609 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/a/b850860b17e7469e8050be37f6e06cda, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T05:45:14,609 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b4ae320d394710486e2563f4c5d712c4 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T05:45:14,610 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/0000000000000000002 2024-11-13T05:45:14,610 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,610 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,613 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4ae320d394710486e2563f4c5d712c4 2024-11-13T05:45:14,616 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/b4ae320d394710486e2563f4c5d712c4/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T05:45:14,617 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4ae320d394710486e2563f4c5d712c4; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60819187, jitterRate=-0.09372349083423615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:14,617 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4ae320d394710486e2563f4c5d712c4: Writing region info on filesystem at 1731476714527Initializing all the Stores at 1731476714529 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476714529Obtaining lock to block concurrent updates at 1731476714566 (+37 ms)Preparing flush snapshotting stores in b4ae320d394710486e2563f4c5d712c4 at 1731476714566Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731476714566Flushing stores of testReplayEditsWrittenIntoWAL,,1731476714106.b4ae320d394710486e2563f4c5d712c4. at 1731476714566Flushing b4ae320d394710486e2563f4c5d712c4/a: creating writer at 1731476714566Flushing b4ae320d394710486e2563f4c5d712c4/a: appending metadata at 1731476714579 (+13 ms)Flushing b4ae320d394710486e2563f4c5d712c4/a: closing flushed file at 1731476714579Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7174640b: reopening flushed file at 1731476714598 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b4ae320d394710486e2563f4c5d712c4 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1731476714609 (+11 ms)Cleaning up temporary data from old regions at 1731476714610 (+1 ms)Region opened successfully at 1731476714617 (+7 ms) 2024-11-13T05:45:14,642 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=383 (was 373) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:38536 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:56938 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:53216 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:50846 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:44216 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=7809 (was 7830) 2024-11-13T05:45:14,652 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=383, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=7809 2024-11-13T05:45:14,669 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:14,671 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:14,672 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:14,675 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29698911, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-29698911, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:14,687 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29698911/hregion-29698911.1731476714675, exclude list is [], retry=0 2024-11-13T05:45:14,690 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:14,691 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:14,691 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:14,718 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29698911/hregion-29698911.1731476714675 2024-11-13T05:45:14,718 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:14,718 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c8e244fbf054b4d4d9e339dff6ac08a0, NAME => 'testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:14,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741854_1030 (size=64) 2024-11-13T05:45:14,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741854_1030 (size=64) 2024-11-13T05:45:14,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741854_1030 (size=64) 2024-11-13T05:45:14,731 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:14,733 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,735 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8e244fbf054b4d4d9e339dff6ac08a0 columnFamilyName a 2024-11-13T05:45:14,736 DEBUG [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:14,737 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(327): Store=c8e244fbf054b4d4d9e339dff6ac08a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:14,737 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,739 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8e244fbf054b4d4d9e339dff6ac08a0 columnFamilyName b 2024-11-13T05:45:14,739 DEBUG [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:14,740 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(327): Store=c8e244fbf054b4d4d9e339dff6ac08a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:14,740 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,743 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8e244fbf054b4d4d9e339dff6ac08a0 columnFamilyName c 2024-11-13T05:45:14,743 DEBUG [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:14,743 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(327): Store=c8e244fbf054b4d4d9e339dff6ac08a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:14,744 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,747 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,747 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,748 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:14,749 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:14,752 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:14,753 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c8e244fbf054b4d4d9e339dff6ac08a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61979229, jitterRate=-0.07643751800060272}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:14,753 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c8e244fbf054b4d4d9e339dff6ac08a0: Writing region info on filesystem at 1731476714731Initializing all the Stores at 1731476714732 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476714732Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476714732Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476714732Cleaning up temporary data from old regions at 1731476714747 (+15 ms)Region opened successfully at 1731476714753 (+6 ms) 2024-11-13T05:45:14,753 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c8e244fbf054b4d4d9e339dff6ac08a0, disabling compactions & flushes 2024-11-13T05:45:14,753 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:14,753 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:14,753 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. after waiting 0 ms 2024-11-13T05:45:14,753 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:14,754 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:14,754 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c8e244fbf054b4d4d9e339dff6ac08a0: Waiting for close lock at 1731476714753Disabling compacts and flushes for region at 1731476714753Disabling writes for close at 1731476714753Writing region close event to WAL at 1731476714754 (+1 ms)Closed at 1731476714754 2024-11-13T05:45:14,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741853_1029 (size=175) 2024-11-13T05:45:14,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741853_1029 (size=175) 2024-11-13T05:45:14,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741853_1029 (size=175) 2024-11-13T05:45:14,760 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:14,760 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29698911:(num 1731476714675) 2024-11-13T05:45:14,761 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:14,763 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:14,775 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763, exclude list is [], retry=0 2024-11-13T05:45:14,779 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:14,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:14,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:14,808 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 2024-11-13T05:45:14,808 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:15,001 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763, size=0 (0bytes) 2024-11-13T05:45:15,001 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 might be still open, length is 0 2024-11-13T05:45:15,001 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 2024-11-13T05:45:15,002 WARN [IPC Server handler 3 on default port 39659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T05:45:15,002 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 after 1ms 2024-11-13T05:45:15,070 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:56960 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56960 dst: /127.0.0.1:46375 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46375 remote=/127.0.0.1:56960]. Total timeout mills is 60000, 59903 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:15,071 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:50880 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50880 dst: /127.0.0.1:42769 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:15,071 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:53270 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41687:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53270 dst: /127.0.0.1:41687 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:15,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741855_1032 (size=497775) 2024-11-13T05:45:15,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741855_1032 (size=497775) 2024-11-13T05:45:15,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741855_1032 (size=497775) 2024-11-13T05:45:18,495 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:45:18,550 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:45:19,003 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 after 4002ms 2024-11-13T05:45:19,035 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:19,036 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 took 4035ms 2024-11-13T05:45:19,041 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731476714763.temp 2024-11-13T05:45:19,071 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000000001-wal.1731476714763.temp 2024-11-13T05:45:19,140 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763; continuing. 2024-11-13T05:45:19,140 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 so closing down 2024-11-13T05:45:19,140 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:19,140 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:19,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741856_1033 (size=497783) 2024-11-13T05:45:19,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741856_1033 (size=497783) 2024-11-13T05:45:19,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741856_1033 (size=497783) 2024-11-13T05:45:19,145 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000000001-wal.1731476714763.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-11-13T05:45:19,147 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000000001-wal.1731476714763.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000003002 2024-11-13T05:45:19,147 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T05:45:19,147 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763, journal: Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763, size=0 (0bytes) at 1731476715001Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000000001-wal.1731476714763.temp at 1731476719071 (+4070 ms)Split 1024 edits, skipped 0 edits. at 1731476719094 (+23 ms)Split 2048 edits, skipped 0 edits. at 1731476719119 (+25 ms)Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 so closing down at 1731476719140 (+21 ms)3 split writer threads finished at 1731476719140Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000000001-wal.1731476714763.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1731476719145 (+5 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000000001-wal.1731476714763.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000003002 at 1731476719147 (+2 ms)Processed 3002 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763, size=0, length=0, corrupted=false, cancelled=false at 1731476719147 2024-11-13T05:45:19,150 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476714763 2024-11-13T05:45:19,151 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000003002 2024-11-13T05:45:19,151 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:19,154 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:19,167 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476719154, exclude list is [], retry=0 2024-11-13T05:45:19,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:19,171 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:19,172 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:19,203 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476719154 2024-11-13T05:45:19,203 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:19,204 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:19,206 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,207 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8e244fbf054b4d4d9e339dff6ac08a0 columnFamilyName a 2024-11-13T05:45:19,207 DEBUG [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:19,208 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(327): Store=c8e244fbf054b4d4d9e339dff6ac08a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:19,208 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,209 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8e244fbf054b4d4d9e339dff6ac08a0 columnFamilyName b 2024-11-13T05:45:19,209 DEBUG [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:19,209 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(327): Store=c8e244fbf054b4d4d9e339dff6ac08a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:19,209 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,210 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8e244fbf054b4d4d9e339dff6ac08a0 columnFamilyName c 2024-11-13T05:45:19,211 DEBUG [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:19,211 INFO [StoreOpener-c8e244fbf054b4d4d9e339dff6ac08a0-1 {}] regionserver.HStore(327): Store=c8e244fbf054b4d4d9e339dff6ac08a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:19,211 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,215 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000003002 2024-11-13T05:45:19,242 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:19,285 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T05:45:19,640 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c8e244fbf054b4d4d9e339dff6ac08a0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T05:45:19,677 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/a/cb64b44862d94920850f40bcaf233b3c is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731476714815/Put/seqid=0 2024-11-13T05:45:19,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741858_1035 (size=50463) 2024-11-13T05:45:19,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741858_1035 (size=50463) 2024-11-13T05:45:19,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741858_1035 (size=50463) 2024-11-13T05:45:19,686 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/a/cb64b44862d94920850f40bcaf233b3c 2024-11-13T05:45:19,695 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/a/cb64b44862d94920850f40bcaf233b3c as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/a/cb64b44862d94920850f40bcaf233b3c 2024-11-13T05:45:19,703 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/a/cb64b44862d94920850f40bcaf233b3c, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T05:45:19,704 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for c8e244fbf054b4d4d9e339dff6ac08a0 in 64ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T05:45:19,720 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T05:45:19,720 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c8e244fbf054b4d4d9e339dff6ac08a0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T05:45:19,728 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/a/3c2fd65b757c4811b748e20432f2db22 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731476714855/Put/seqid=0 2024-11-13T05:45:19,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741859_1036 (size=20072) 2024-11-13T05:45:19,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741859_1036 (size=20072) 2024-11-13T05:45:19,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741859_1036 (size=20072) 2024-11-13T05:45:19,738 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/a/3c2fd65b757c4811b748e20432f2db22 2024-11-13T05:45:19,763 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/b/8c1a7881e85840e28769c855a577c81a is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731476714881/Put/seqid=0 2024-11-13T05:45:19,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741860_1037 (size=35835) 2024-11-13T05:45:19,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741860_1037 (size=35835) 2024-11-13T05:45:19,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741860_1037 (size=35835) 2024-11-13T05:45:19,771 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/b/8c1a7881e85840e28769c855a577c81a 2024-11-13T05:45:19,779 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/a/3c2fd65b757c4811b748e20432f2db22 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/a/3c2fd65b757c4811b748e20432f2db22 2024-11-13T05:45:19,788 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/a/3c2fd65b757c4811b748e20432f2db22, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T05:45:19,791 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/b/8c1a7881e85840e28769c855a577c81a as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/b/8c1a7881e85840e28769c855a577c81a 2024-11-13T05:45:19,798 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/b/8c1a7881e85840e28769c855a577c81a, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T05:45:19,799 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for c8e244fbf054b4d4d9e339dff6ac08a0 in 79ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T05:45:19,814 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T05:45:19,814 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c8e244fbf054b4d4d9e339dff6ac08a0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T05:45:19,821 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/b/a42dd32b4e084b3e911c14ec89552733 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731476714898/Put/seqid=0 2024-11-13T05:45:19,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741861_1038 (size=35082) 2024-11-13T05:45:19,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741861_1038 (size=35082) 2024-11-13T05:45:19,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741861_1038 (size=35082) 2024-11-13T05:45:19,831 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/b/a42dd32b4e084b3e911c14ec89552733 2024-11-13T05:45:19,856 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/c/4db4c1e4d3a0479585a3522e96c3e7df is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731476714924/Put/seqid=0 2024-11-13T05:45:19,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741862_1039 (size=20825) 2024-11-13T05:45:19,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741862_1039 (size=20825) 2024-11-13T05:45:19,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741862_1039 (size=20825) 2024-11-13T05:45:19,866 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/c/4db4c1e4d3a0479585a3522e96c3e7df 2024-11-13T05:45:19,874 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/b/a42dd32b4e084b3e911c14ec89552733 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/b/a42dd32b4e084b3e911c14ec89552733 2024-11-13T05:45:19,883 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/b/a42dd32b4e084b3e911c14ec89552733, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T05:45:19,885 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/c/4db4c1e4d3a0479585a3522e96c3e7df as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/c/4db4c1e4d3a0479585a3522e96c3e7df 2024-11-13T05:45:19,892 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/c/4db4c1e4d3a0479585a3522e96c3e7df, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T05:45:19,892 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for c8e244fbf054b4d4d9e339dff6ac08a0 in 78ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T05:45:19,905 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731476714972/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:19,908 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000003002 2024-11-13T05:45:19,908 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T05:45:19,909 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c8e244fbf054b4d4d9e339dff6ac08a0 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T05:45:19,921 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/c/6ae0763cbb614b3b9a0cc32bbccc71f9 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731476714930/Put/seqid=0 2024-11-13T05:45:19,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741863_1040 (size=50301) 2024-11-13T05:45:19,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741863_1040 (size=50301) 2024-11-13T05:45:19,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741863_1040 (size=50301) 2024-11-13T05:45:19,933 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/c/6ae0763cbb614b3b9a0cc32bbccc71f9 2024-11-13T05:45:19,940 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ae0763cbb614b3b9a0cc32bbccc71f9 2024-11-13T05:45:19,942 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/.tmp/c/6ae0763cbb614b3b9a0cc32bbccc71f9 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/c/6ae0763cbb614b3b9a0cc32bbccc71f9 2024-11-13T05:45:19,950 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ae0763cbb614b3b9a0cc32bbccc71f9 2024-11-13T05:45:19,950 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/c/6ae0763cbb614b3b9a0cc32bbccc71f9, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T05:45:19,951 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for c8e244fbf054b4d4d9e339dff6ac08a0 in 42ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T05:45:19,952 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/0000000000000003002 2024-11-13T05:45:19,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,955 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T05:45:19,958 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c8e244fbf054b4d4d9e339dff6ac08a0 2024-11-13T05:45:19,961 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenIntoWAL/c8e244fbf054b4d4d9e339dff6ac08a0/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T05:45:19,962 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c8e244fbf054b4d4d9e339dff6ac08a0; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66980232, jitterRate=-0.0019167661666870117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T05:45:19,963 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c8e244fbf054b4d4d9e339dff6ac08a0: Writing region info on filesystem at 1731476719204Initializing all the Stores at 1731476719205 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476719205Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476719205Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476719205Cleaning up temporary data from old regions at 1731476719954 (+749 ms)Region opened successfully at 1731476719963 (+9 ms) 2024-11-13T05:45:20,025 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c8e244fbf054b4d4d9e339dff6ac08a0, disabling compactions & flushes 2024-11-13T05:45:20,025 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:20,026 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:20,026 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. after waiting 0 ms 2024-11-13T05:45:20,026 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:20,028 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731476714669.c8e244fbf054b4d4d9e339dff6ac08a0. 2024-11-13T05:45:20,028 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c8e244fbf054b4d4d9e339dff6ac08a0: Waiting for close lock at 1731476720025Disabling compacts and flushes for region at 1731476720025Disabling writes for close at 1731476720026 (+1 ms)Writing region close event to WAL at 1731476720028 (+2 ms)Closed at 1731476720028 2024-11-13T05:45:20,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741857_1034 (size=175) 2024-11-13T05:45:20,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741857_1034 (size=175) 2024-11-13T05:45:20,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741857_1034 (size=175) 2024-11-13T05:45:20,034 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:20,035 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731476719154) 2024-11-13T05:45:20,049 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=397 (was 383) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2110679462_22 at /127.0.0.1:57000 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39659 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2110679462_22 at /127.0.0.1:53290 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2110679462_22 at /127.0.0.1:50918 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2110679462_22 at /127.0.0.1:50948 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:35543 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35543 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:35781 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35781 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:39659 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 343) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7709 (was 7809) 2024-11-13T05:45:20,061 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=397, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=355, ProcessCount=11, AvailableMemoryMB=7708 2024-11-13T05:45:20,077 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:20,080 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:20,081 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:20,085 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70588306, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-70588306, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:20,098 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70588306/hregion-70588306.1731476720085, exclude list is [], retry=0 2024-11-13T05:45:20,101 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:20,101 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:20,101 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:20,131 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70588306/hregion-70588306.1731476720085 2024-11-13T05:45:20,131 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:20,132 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d1b8a04ce438b1ab648624f9c29e514c, NAME => 'test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:20,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741865_1042 (size=43) 2024-11-13T05:45:20,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741865_1042 (size=43) 2024-11-13T05:45:20,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741865_1042 (size=43) 2024-11-13T05:45:20,144 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:20,145 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,147 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1b8a04ce438b1ab648624f9c29e514c columnFamilyName a 2024-11-13T05:45:20,147 DEBUG [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:20,148 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(327): Store=d1b8a04ce438b1ab648624f9c29e514c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:20,148 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,150 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1b8a04ce438b1ab648624f9c29e514c columnFamilyName b 2024-11-13T05:45:20,150 DEBUG [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:20,150 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(327): Store=d1b8a04ce438b1ab648624f9c29e514c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:20,150 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,152 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1b8a04ce438b1ab648624f9c29e514c columnFamilyName c 2024-11-13T05:45:20,152 DEBUG [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:20,153 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(327): Store=d1b8a04ce438b1ab648624f9c29e514c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:20,153 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,155 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,155 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,156 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:20,157 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,160 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:20,161 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1b8a04ce438b1ab648624f9c29e514c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72152525, jitterRate=0.07515640556812286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:20,162 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1b8a04ce438b1ab648624f9c29e514c: Writing region info on filesystem at 1731476720144Initializing all the Stores at 1731476720145 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476720145Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476720145Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476720145Cleaning up temporary data from old regions at 1731476720155 (+10 ms)Region opened successfully at 1731476720162 (+7 ms) 2024-11-13T05:45:20,162 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d1b8a04ce438b1ab648624f9c29e514c, disabling compactions & flushes 2024-11-13T05:45:20,162 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:20,162 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:20,162 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. after waiting 0 ms 2024-11-13T05:45:20,162 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:20,162 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:20,163 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d1b8a04ce438b1ab648624f9c29e514c: Waiting for close lock at 1731476720162Disabling compacts and flushes for region at 1731476720162Disabling writes for close at 1731476720162Writing region close event to WAL at 1731476720162Closed at 1731476720162 2024-11-13T05:45:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741864_1041 (size=175) 2024-11-13T05:45:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741864_1041 (size=175) 2024-11-13T05:45:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741864_1041 (size=175) 2024-11-13T05:45:20,169 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:20,169 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70588306:(num 1731476720085) 2024-11-13T05:45:20,169 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:20,171 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:20,188 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171, exclude list is [], retry=0 2024-11-13T05:45:20,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:20,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:20,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:20,222 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171 2024-11-13T05:45:20,223 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:20,286 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:45:20,287 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:20,289 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:45:20,289 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:45:20,290 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:45:20,290 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:20,291 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T05:45:20,291 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:20,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741866_1043 (size=371388) 2024-11-13T05:45:20,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741866_1043 (size=371388) 2024-11-13T05:45:20,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741866_1043 (size=371388) 2024-11-13T05:45:20,395 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171, size=362.7 K (371388bytes) 2024-11-13T05:45:20,395 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171 2024-11-13T05:45:20,396 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171 after 1ms 2024-11-13T05:45:20,427 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:20,429 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171 took 34ms 2024-11-13T05:45:20,433 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731476720171.temp 2024-11-13T05:45:20,474 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171 so closing down 2024-11-13T05:45:20,474 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:20,477 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000000001-wal.1731476720171.temp 2024-11-13T05:45:20,498 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:20,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741867_1044 (size=371388) 2024-11-13T05:45:20,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741867_1044 (size=371388) 2024-11-13T05:45:20,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741867_1044 (size=371388) 2024-11-13T05:45:20,504 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000000001-wal.1731476720171.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-13T05:45:20,506 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000000001-wal.1731476720171.temp to hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003000 2024-11-13T05:45:20,506 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-13T05:45:20,506 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171, journal: Splitting hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171, size=362.7 K (371388bytes) at 1731476720395Split 1024 edits, skipped 0 edits. at 1731476720442 (+47 ms)Split 2048 edits, skipped 0 edits. at 1731476720459 (+17 ms)Finishing writing output for hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171 so closing down at 1731476720474 (+15 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000000001-wal.1731476720171.temp at 1731476720477 (+3 ms)3 split writer threads finished at 1731476720498 (+21 ms)Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000000001-wal.1731476720171.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1731476720504 (+6 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000000001-wal.1731476720171.temp to hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003000 at 1731476720506 (+2 ms)Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731476720506 2024-11-13T05:45:20,508 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720171 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476720171 2024-11-13T05:45:20,509 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003000 2024-11-13T05:45:20,510 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:20,512 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:20,528 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512, exclude list is [], retry=0 2024-11-13T05:45:20,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:20,532 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:20,532 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:20,562 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512 2024-11-13T05:45:20,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:20,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741868_1045 (size=371515) 2024-11-13T05:45:20,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741868_1045 (size=371515) 2024-11-13T05:45:20,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741868_1045 (size=371515) 2024-11-13T05:45:20,742 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512, size=362.8 K (371515bytes) 2024-11-13T05:45:20,743 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512 2024-11-13T05:45:20,743 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512 after 0ms 2024-11-13T05:45:20,775 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:20,777 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512 took 35ms 2024-11-13T05:45:20,781 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731476720512.temp 2024-11-13T05:45:20,837 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512 so closing down 2024-11-13T05:45:20,838 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:20,843 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003001-wal.1731476720512.temp 2024-11-13T05:45:20,853 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:20,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741869_1046 (size=371515) 2024-11-13T05:45:20,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741869_1046 (size=371515) 2024-11-13T05:45:20,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741869_1046 (size=371515) 2024-11-13T05:45:20,865 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003001-wal.1731476720512.temp (wrote 3000 edits, skipped 0 edits in 9 ms) 2024-11-13T05:45:20,867 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003001-wal.1731476720512.temp to hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000006000 2024-11-13T05:45:20,867 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 90 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-13T05:45:20,867 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512, journal: Splitting hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512, size=362.8 K (371515bytes) at 1731476720742Split 1024 edits, skipped 0 edits. at 1731476720816 (+74 ms)Split 2048 edits, skipped 0 edits. at 1731476720826 (+10 ms)Finishing writing output for hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512 so closing down at 1731476720837 (+11 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003001-wal.1731476720512.temp at 1731476720843 (+6 ms)3 split writer threads finished at 1731476720853 (+10 ms)Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003001-wal.1731476720512.temp (wrote 3000 edits, skipped 0 edits in 9 ms) at 1731476720865 (+12 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003001-wal.1731476720512.temp to hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000006000 at 1731476720867 (+2 ms)Processed 3000 edits across 1 Regions in 90 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731476720867 2024-11-13T05:45:20,869 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720512 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476720512 2024-11-13T05:45:20,871 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000006000 2024-11-13T05:45:20,871 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:20,873 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/test2727-manual,16010,1731476720076, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:20,892 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720874, exclude list is [], retry=0 2024-11-13T05:45:20,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:20,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:20,897 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:20,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731476720076/wal.1731476720874 2024-11-13T05:45:20,926 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:20,926 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d1b8a04ce438b1ab648624f9c29e514c, NAME => 'test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:20,926 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:20,926 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,926 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,928 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,930 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1b8a04ce438b1ab648624f9c29e514c columnFamilyName a 2024-11-13T05:45:20,930 DEBUG [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:20,931 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(327): Store=d1b8a04ce438b1ab648624f9c29e514c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:20,931 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,932 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1b8a04ce438b1ab648624f9c29e514c columnFamilyName b 2024-11-13T05:45:20,932 DEBUG [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:20,933 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(327): Store=d1b8a04ce438b1ab648624f9c29e514c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:20,933 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,934 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1b8a04ce438b1ab648624f9c29e514c columnFamilyName c 2024-11-13T05:45:20,934 DEBUG [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:20,935 INFO [StoreOpener-d1b8a04ce438b1ab648624f9c29e514c-1 {}] regionserver.HStore(327): Store=d1b8a04ce438b1ab648624f9c29e514c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:20,935 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,937 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,940 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:20,941 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003000 2024-11-13T05:45:20,969 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:20,997 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003000 2024-11-13T05:45:20,998 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000006000 2024-11-13T05:45:21,026 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:21,052 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000006000 2024-11-13T05:45:21,052 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d1b8a04ce438b1ab648624f9c29e514c 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T05:45:21,072 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/a/9ff3e2a3a01f48baa90e2b7495ccc793 is 41, key is test2727/a:100/1731476720567/Put/seqid=0 2024-11-13T05:45:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741871_1048 (size=84227) 2024-11-13T05:45:21,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741871_1048 (size=84227) 2024-11-13T05:45:21,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741871_1048 (size=84227) 2024-11-13T05:45:21,082 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/a/9ff3e2a3a01f48baa90e2b7495ccc793 2024-11-13T05:45:21,112 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/b/77078f29355e4e6686f29d381ba0206a is 41, key is test2727/b:100/1731476720614/Put/seqid=0 2024-11-13T05:45:21,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741872_1049 (size=84609) 2024-11-13T05:45:21,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741872_1049 (size=84609) 2024-11-13T05:45:21,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741872_1049 (size=84609) 2024-11-13T05:45:21,119 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/b/77078f29355e4e6686f29d381ba0206a 2024-11-13T05:45:21,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/c/5e3a636e36d34fb3a92a77995a55fdda is 41, key is test2727/c:100/1731476720669/Put/seqid=0 2024-11-13T05:45:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741873_1050 (size=84609) 2024-11-13T05:45:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741873_1050 (size=84609) 2024-11-13T05:45:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741873_1050 (size=84609) 2024-11-13T05:45:21,152 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/c/5e3a636e36d34fb3a92a77995a55fdda 2024-11-13T05:45:21,158 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/a/9ff3e2a3a01f48baa90e2b7495ccc793 as hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/a/9ff3e2a3a01f48baa90e2b7495ccc793 2024-11-13T05:45:21,164 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/a/9ff3e2a3a01f48baa90e2b7495ccc793, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T05:45:21,166 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/b/77078f29355e4e6686f29d381ba0206a as hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/b/77078f29355e4e6686f29d381ba0206a 2024-11-13T05:45:21,173 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/b/77078f29355e4e6686f29d381ba0206a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T05:45:21,174 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/.tmp/c/5e3a636e36d34fb3a92a77995a55fdda as hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/c/5e3a636e36d34fb3a92a77995a55fdda 2024-11-13T05:45:21,181 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/c/5e3a636e36d34fb3a92a77995a55fdda, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T05:45:21,181 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d1b8a04ce438b1ab648624f9c29e514c in 129ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T05:45:21,182 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000003000 2024-11-13T05:45:21,183 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/0000000000000006000 2024-11-13T05:45:21,184 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:21,185 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:21,185 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:21,187 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1b8a04ce438b1ab648624f9c29e514c 2024-11-13T05:45:21,190 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/test2727/d1b8a04ce438b1ab648624f9c29e514c/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T05:45:21,191 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1b8a04ce438b1ab648624f9c29e514c; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64508882, jitterRate=-0.03874275088310242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:21,192 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1b8a04ce438b1ab648624f9c29e514c: Writing region info on filesystem at 1731476720927Initializing all the Stores at 1731476720928 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476720928Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476720928Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476720928Obtaining lock to block concurrent updates at 1731476721052 (+124 ms)Preparing flush snapshotting stores in d1b8a04ce438b1ab648624f9c29e514c at 1731476721052Finished memstore snapshotting test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731476721052Flushing stores of test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. at 1731476721052Flushing d1b8a04ce438b1ab648624f9c29e514c/a: creating writer at 1731476721053 (+1 ms)Flushing d1b8a04ce438b1ab648624f9c29e514c/a: appending metadata at 1731476721072 (+19 ms)Flushing d1b8a04ce438b1ab648624f9c29e514c/a: closing flushed file at 1731476721072Flushing d1b8a04ce438b1ab648624f9c29e514c/b: creating writer at 1731476721089 (+17 ms)Flushing d1b8a04ce438b1ab648624f9c29e514c/b: appending metadata at 1731476721110 (+21 ms)Flushing d1b8a04ce438b1ab648624f9c29e514c/b: closing flushed file at 1731476721110Flushing d1b8a04ce438b1ab648624f9c29e514c/c: creating writer at 1731476721126 (+16 ms)Flushing d1b8a04ce438b1ab648624f9c29e514c/c: appending metadata at 1731476721145 (+19 ms)Flushing d1b8a04ce438b1ab648624f9c29e514c/c: closing flushed file at 1731476721145Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@beac4: reopening flushed file at 1731476721157 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9f8d024: reopening flushed file at 1731476721164 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56560e3a: reopening flushed file at 1731476721173 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d1b8a04ce438b1ab648624f9c29e514c in 129ms, sequenceid=6000, compaction requested=false; wal=null at 1731476721181 (+8 ms)Cleaning up temporary data from old regions at 1731476721185 (+4 ms)Region opened successfully at 1731476721192 (+7 ms) 2024-11-13T05:45:21,193 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T05:45:21,193 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d1b8a04ce438b1ab648624f9c29e514c, disabling compactions & flushes 2024-11-13T05:45:21,193 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:21,193 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:21,193 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. after waiting 0 ms 2024-11-13T05:45:21,193 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:21,195 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731476720078.d1b8a04ce438b1ab648624f9c29e514c. 2024-11-13T05:45:21,195 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d1b8a04ce438b1ab648624f9c29e514c: Waiting for close lock at 1731476721193Disabling compacts and flushes for region at 1731476721193Disabling writes for close at 1731476721193Writing region close event to WAL at 1731476721194 (+1 ms)Closed at 1731476721194 2024-11-13T05:45:21,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741870_1047 (size=175) 2024-11-13T05:45:21,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741870_1047 (size=175) 2024-11-13T05:45:21,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741870_1047 (size=175) 2024-11-13T05:45:21,201 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:21,201 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731476720874) 2024-11-13T05:45:21,213 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=401 (was 397) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:57106 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:50948 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:53446 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 355), ProcessCount=11 (was 11), AvailableMemoryMB=7480 (was 7708) 2024-11-13T05:45:21,224 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=355, ProcessCount=11, AvailableMemoryMB=7480 2024-11-13T05:45:21,239 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:21,246 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:21,246 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731476721246 2024-11-13T05:45:21,279 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246 2024-11-13T05:45:21,283 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:21,284 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dba774249b4e2e883dd653b48e785959, NAME => 'testSequentialEditLogSeqNum,,1731476721240.dba774249b4e2e883dd653b48e785959.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:21,285 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731476721240.dba774249b4e2e883dd653b48e785959.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:21,285 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,285 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,286 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959 doesn't exist for region: dba774249b4e2e883dd653b48e785959 on table testSequentialEditLogSeqNum 2024-11-13T05:45:21,286 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: dba774249b4e2e883dd653b48e785959 on table testSequentialEditLogSeqNum 2024-11-13T05:45:21,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741875_1052 (size=62) 2024-11-13T05:45:21,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741875_1052 (size=62) 2024-11-13T05:45:21,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741875_1052 (size=62) 2024-11-13T05:45:21,298 INFO [StoreOpener-dba774249b4e2e883dd653b48e785959-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,299 INFO [StoreOpener-dba774249b4e2e883dd653b48e785959-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dba774249b4e2e883dd653b48e785959 columnFamilyName a 2024-11-13T05:45:21,300 DEBUG [StoreOpener-dba774249b4e2e883dd653b48e785959-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:21,300 INFO [StoreOpener-dba774249b4e2e883dd653b48e785959-1 {}] regionserver.HStore(327): Store=dba774249b4e2e883dd653b48e785959/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:21,300 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,301 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,301 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,302 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,302 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,304 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dba774249b4e2e883dd653b48e785959 2024-11-13T05:45:21,306 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:21,306 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dba774249b4e2e883dd653b48e785959; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60416020, jitterRate=-0.09973114728927612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:21,307 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dba774249b4e2e883dd653b48e785959: Writing region info on filesystem at 1731476721285Initializing all the Stores at 1731476721297 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476721297Cleaning up temporary data from old regions at 1731476721302 (+5 ms)Region opened successfully at 1731476721307 (+5 ms) 2024-11-13T05:45:21,318 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dba774249b4e2e883dd653b48e785959 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T05:45:21,340 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/.tmp/a/2742ccf90b974d29aeb34ed8f2f50658 is 81, key is testSequentialEditLogSeqNum/a:x0/1731476721307/Put/seqid=0 2024-11-13T05:45:21,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741876_1053 (size=5833) 2024-11-13T05:45:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741876_1053 (size=5833) 2024-11-13T05:45:21,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741876_1053 (size=5833) 2024-11-13T05:45:21,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/.tmp/a/2742ccf90b974d29aeb34ed8f2f50658 2024-11-13T05:45:21,359 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/.tmp/a/2742ccf90b974d29aeb34ed8f2f50658 as hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/a/2742ccf90b974d29aeb34ed8f2f50658 2024-11-13T05:45:21,366 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/a/2742ccf90b974d29aeb34ed8f2f50658, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T05:45:21,367 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for dba774249b4e2e883dd653b48e785959 in 49ms, sequenceid=13, compaction requested=false 2024-11-13T05:45:21,367 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dba774249b4e2e883dd653b48e785959: 2024-11-13T05:45:21,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:45:21,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:45:21,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:45:21,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:45:21,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T05:45:21,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741874_1051 (size=3256) 2024-11-13T05:45:21,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741874_1051 (size=3256) 2024-11-13T05:45:21,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741874_1051 (size=3256) 2024-11-13T05:45:21,392 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246, size=3.2 K (3256bytes) 2024-11-13T05:45:21,392 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246 2024-11-13T05:45:21,393 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246 after 1ms 2024-11-13T05:45:21,419 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:21,419 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246 took 27ms 2024-11-13T05:45:21,423 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246 so closing down 2024-11-13T05:45:21,423 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:21,424 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731476721246.temp 2024-11-13T05:45:21,451 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000003-wal.1731476721246.temp 2024-11-13T05:45:21,452 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:21,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741877_1054 (size=2650) 2024-11-13T05:45:21,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741877_1054 (size=2650) 2024-11-13T05:45:21,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741877_1054 (size=2650) 2024-11-13T05:45:21,462 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000003-wal.1731476721246.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:21,463 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000003-wal.1731476721246.temp to hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000020 2024-11-13T05:45:21,463 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-13T05:45:21,464 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246, journal: Splitting hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246, size=3.2 K (3256bytes) at 1731476721392Finishing writing output for hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246 so closing down at 1731476721423 (+31 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000003-wal.1731476721246.temp at 1731476721451 (+28 ms)3 split writer threads finished at 1731476721452 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000003-wal.1731476721246.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731476721462 (+10 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000003-wal.1731476721246.temp to hdfs://localhost:39659/hbase/data/default/testSequentialEditLogSeqNum/dba774249b4e2e883dd653b48e785959/recovered.edits/0000000000000000020 at 1731476721463 (+1 ms)Processed 17 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:39659/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731476721238/wal.1731476721246, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731476721463 2024-11-13T05:45:21,476 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:50948 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:53446 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 355), ProcessCount=11 (was 11), AvailableMemoryMB=7477 (was 7480) 2024-11-13T05:45:21,486 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=406, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=355, ProcessCount=11, AvailableMemoryMB=7476 2024-11-13T05:45:21,502 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:21,504 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:21,535 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:21,539 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72497129, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-72497129, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:21,555 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72497129/hregion-72497129.1731476721540, exclude list is [], retry=0 2024-11-13T05:45:21,558 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:21,558 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:21,559 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:21,584 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72497129/hregion-72497129.1731476721540 2024-11-13T05:45:21,585 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:21,585 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8972f8d64e1834406a003adab20f9491, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:21,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741879_1056 (size=70) 2024-11-13T05:45:21,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741879_1056 (size=70) 2024-11-13T05:45:21,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741879_1056 (size=70) 2024-11-13T05:45:21,598 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:21,600 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,601 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName a 2024-11-13T05:45:21,601 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:21,602 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:21,602 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,603 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName b 2024-11-13T05:45:21,604 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:21,604 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:21,604 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,606 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName c 2024-11-13T05:45:21,606 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:21,607 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:21,607 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,608 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,608 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,609 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,609 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,610 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:21,611 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,613 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:21,614 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8972f8d64e1834406a003adab20f9491; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68235677, jitterRate=0.01679082214832306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:21,614 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8972f8d64e1834406a003adab20f9491: Writing region info on filesystem at 1731476721598Initializing all the Stores at 1731476721599 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476721599Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476721599Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476721599Cleaning up temporary data from old regions at 1731476721609 (+10 ms)Region opened successfully at 1731476721614 (+5 ms) 2024-11-13T05:45:21,614 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8972f8d64e1834406a003adab20f9491, disabling compactions & flushes 2024-11-13T05:45:21,614 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:21,615 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:21,615 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. after waiting 0 ms 2024-11-13T05:45:21,615 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:21,615 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:21,615 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8972f8d64e1834406a003adab20f9491: Waiting for close lock at 1731476721614Disabling compacts and flushes for region at 1731476721614Disabling writes for close at 1731476721615 (+1 ms)Writing region close event to WAL at 1731476721615Closed at 1731476721615 2024-11-13T05:45:21,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741878_1055 (size=175) 2024-11-13T05:45:21,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741878_1055 (size=175) 2024-11-13T05:45:21,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741878_1055 (size=175) 2024-11-13T05:45:21,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:21,623 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72497129:(num 1731476721540) 2024-11-13T05:45:21,623 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:21,625 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:21,638 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626, exclude list is [], retry=0 2024-11-13T05:45:21,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:21,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:21,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:21,669 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 2024-11-13T05:45:21,670 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:21,670 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8972f8d64e1834406a003adab20f9491, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:21,670 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:21,670 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,670 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,672 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,673 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName a 2024-11-13T05:45:21,673 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:21,674 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:21,674 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,675 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName b 2024-11-13T05:45:21,675 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:21,675 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:21,675 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,676 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName c 2024-11-13T05:45:21,676 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:21,677 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:21,677 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,677 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,679 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,680 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,680 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,680 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:21,682 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:21,682 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8972f8d64e1834406a003adab20f9491; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66764476, jitterRate=-0.005131781101226807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:21,683 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8972f8d64e1834406a003adab20f9491: Writing region info on filesystem at 1731476721670Initializing all the Stores at 1731476721671 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476721671Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476721672 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476721672Cleaning up temporary data from old regions at 1731476721680 (+8 ms)Region opened successfully at 1731476721683 (+3 ms) 2024-11-13T05:45:21,687 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731476721686/Put/seqid=0 2024-11-13T05:45:21,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741881_1058 (size=4826) 2024-11-13T05:45:21,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741881_1058 (size=4826) 2024-11-13T05:45:21,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741881_1058 (size=4826) 2024-11-13T05:45:21,695 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39659/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 8972f8d64e1834406a003adab20f9491/a 2024-11-13T05:45:21,705 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T05:45:21,705 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T05:45:21,705 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8972f8d64e1834406a003adab20f9491: 2024-11-13T05:45:21,707 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/a/fccb55ded0fb41caa77a2b3c4c058b19_SeqId_3_ 2024-11-13T05:45:21,708 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39659/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8972f8d64e1834406a003adab20f9491/a as hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/a/fccb55ded0fb41caa77a2b3c4c058b19_SeqId_3_ - updating store file list. 2024-11-13T05:45:21,714 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for fccb55ded0fb41caa77a2b3c4c058b19_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:21,715 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/a/fccb55ded0fb41caa77a2b3c4c058b19_SeqId_3_ into 8972f8d64e1834406a003adab20f9491/a 2024-11-13T05:45:21,715 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39659/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 8972f8d64e1834406a003adab20f9491/a (new location: hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/a/fccb55ded0fb41caa77a2b3c4c058b19_SeqId_3_) 2024-11-13T05:45:21,756 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626, size=0 (0bytes) 2024-11-13T05:45:21,756 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 might be still open, length is 0 2024-11-13T05:45:21,756 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 2024-11-13T05:45:21,757 WARN [IPC Server handler 4 on default port 39659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T05:45:21,757 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 after 1ms 2024-11-13T05:45:24,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:51096 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51096 dst: /127.0.0.1:42769 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42769 remote=/127.0.0.1:51096]. Total timeout mills is 60000, 57656 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:24,066 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:57230 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57230 dst: /127.0.0.1:46375 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:24,066 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:53562 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41687:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53562 dst: /127.0.0.1:41687 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:24,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741880_1059 (size=606) 2024-11-13T05:45:24,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741880_1059 (size=606) 2024-11-13T05:45:25,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 after 4002ms 2024-11-13T05:45:25,788 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:25,788 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 took 4032ms 2024-11-13T05:45:25,790 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626; continuing. 2024-11-13T05:45:25,790 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 so closing down 2024-11-13T05:45:25,790 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:25,792 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731476721626.temp 2024-11-13T05:45:25,816 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005-wal.1731476721626.temp 2024-11-13T05:45:25,817 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:25,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741882_1060 (size=329) 2024-11-13T05:45:25,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741882_1060 (size=329) 2024-11-13T05:45:25,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741882_1060 (size=329) 2024-11-13T05:45:25,824 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005-wal.1731476721626.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:25,825 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005-wal.1731476721626.temp to hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005 2024-11-13T05:45:25,826 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T05:45:25,826 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626, journal: Splitting hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626, size=0 (0bytes) at 1731476721756Finishing writing output for hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 so closing down at 1731476725790 (+4034 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005-wal.1731476721626.temp at 1731476725817 (+27 ms)3 split writer threads finished at 1731476725817Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005-wal.1731476721626.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731476725824 (+7 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005-wal.1731476721626.temp to hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005 at 1731476725825 (+1 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626, size=0, length=0, corrupted=false, cancelled=false at 1731476725826 (+1 ms) 2024-11-13T05:45:25,827 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476721626 2024-11-13T05:45:25,828 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005 2024-11-13T05:45:25,828 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:25,830 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:25,845 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476725830, exclude list is [], retry=0 2024-11-13T05:45:25,848 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:25,848 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:25,849 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:25,873 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476725830 2024-11-13T05:45:25,874 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:25,874 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8972f8d64e1834406a003adab20f9491, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:25,874 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:25,874 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,874 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,876 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,877 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName a 2024-11-13T05:45:25,877 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:25,884 DEBUG [StoreFileOpener-8972f8d64e1834406a003adab20f9491-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for fccb55ded0fb41caa77a2b3c4c058b19_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:25,884 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/a/fccb55ded0fb41caa77a2b3c4c058b19_SeqId_3_ 2024-11-13T05:45:25,885 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:25,885 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,886 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName b 2024-11-13T05:45:25,886 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:25,887 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:25,887 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,888 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8972f8d64e1834406a003adab20f9491 columnFamilyName c 2024-11-13T05:45:25,888 DEBUG [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:25,888 INFO [StoreOpener-8972f8d64e1834406a003adab20f9491-1 {}] regionserver.HStore(327): Store=8972f8d64e1834406a003adab20f9491/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:25,888 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,889 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,891 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005 2024-11-13T05:45:25,917 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:25,919 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005 2024-11-13T05:45:25,919 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8972f8d64e1834406a003adab20f9491 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T05:45:25,933 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/.tmp/a/3d77a40d8d7b4db38dee64fa22a70db2 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731476721721/Put/seqid=0 2024-11-13T05:45:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741884_1062 (size=5149) 2024-11-13T05:45:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741884_1062 (size=5149) 2024-11-13T05:45:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741884_1062 (size=5149) 2024-11-13T05:45:25,940 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/.tmp/a/3d77a40d8d7b4db38dee64fa22a70db2 2024-11-13T05:45:25,946 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/.tmp/a/3d77a40d8d7b4db38dee64fa22a70db2 as hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/a/3d77a40d8d7b4db38dee64fa22a70db2 2024-11-13T05:45:25,952 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/a/3d77a40d8d7b4db38dee64fa22a70db2, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T05:45:25,952 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8972f8d64e1834406a003adab20f9491 in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T05:45:25,953 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/0000000000000000005 2024-11-13T05:45:25,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,955 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:25,956 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8972f8d64e1834406a003adab20f9491 2024-11-13T05:45:25,959 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/8972f8d64e1834406a003adab20f9491/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T05:45:25,959 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8972f8d64e1834406a003adab20f9491; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74618190, jitterRate=0.11189767718315125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:25,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8972f8d64e1834406a003adab20f9491: Writing region info on filesystem at 1731476725875Initializing all the Stores at 1731476725875Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476725876 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476725876Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476725876Obtaining lock to block concurrent updates at 1731476725919 (+43 ms)Preparing flush snapshotting stores in 8972f8d64e1834406a003adab20f9491 at 1731476725919Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731476725919Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. at 1731476725919Flushing 8972f8d64e1834406a003adab20f9491/a: creating writer at 1731476725919Flushing 8972f8d64e1834406a003adab20f9491/a: appending metadata at 1731476725932 (+13 ms)Flushing 8972f8d64e1834406a003adab20f9491/a: closing flushed file at 1731476725933 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4516ca5a: reopening flushed file at 1731476725945 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 8972f8d64e1834406a003adab20f9491 in 33ms, sequenceid=5, compaction requested=false; wal=null at 1731476725952 (+7 ms)Cleaning up temporary data from old regions at 1731476725954 (+2 ms)Region opened successfully at 1731476725960 (+6 ms) 2024-11-13T05:45:25,964 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8972f8d64e1834406a003adab20f9491, disabling compactions & flushes 2024-11-13T05:45:25,964 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:25,964 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:25,964 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. after waiting 0 ms 2024-11-13T05:45:25,964 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:25,965 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731476721503.8972f8d64e1834406a003adab20f9491. 2024-11-13T05:45:25,965 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8972f8d64e1834406a003adab20f9491: Waiting for close lock at 1731476725963Disabling compacts and flushes for region at 1731476725963Disabling writes for close at 1731476725964 (+1 ms)Writing region close event to WAL at 1731476725964Closed at 1731476725964 2024-11-13T05:45:25,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741883_1061 (size=175) 2024-11-13T05:45:25,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741883_1061 (size=175) 2024-11-13T05:45:25,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741883_1061 (size=175) 2024-11-13T05:45:25,970 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:25,970 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731476725830) 2024-11-13T05:45:25,981 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408 (was 406) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:39659 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39659 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-353199108_22 at /127.0.0.1:54718 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-353199108_22 at /127.0.0.1:38852 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-353199108_22 at /127.0.0.1:44354 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=327 (was 355), ProcessCount=11 (was 11), AvailableMemoryMB=7475 (was 7476) 2024-11-13T05:45:25,993 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=327, ProcessCount=11, AvailableMemoryMB=7475 2024-11-13T05:45:26,011 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:26,015 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:45:26,019 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:26,021 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26bc59d0 2024-11-13T05:45:26,022 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:45:26,024 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:45:26,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:45:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T05:45:26,036 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:45:26,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T05:45:26,039 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:26,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:45:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:45:26,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741885_1063 (size=694) 2024-11-13T05:45:26,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741885_1063 (size=694) 2024-11-13T05:45:26,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741885_1063 (size=694) 2024-11-13T05:45:26,054 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06e18f488ac3dcef5cfa0c667ec59dce, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d 2024-11-13T05:45:26,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741886_1064 (size=77) 2024-11-13T05:45:26,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741886_1064 (size=77) 2024-11-13T05:45:26,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741886_1064 (size=77) 2024-11-13T05:45:26,067 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:26,067 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 06e18f488ac3dcef5cfa0c667ec59dce, disabling compactions & flushes 2024-11-13T05:45:26,067 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,067 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,067 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. after waiting 0 ms 2024-11-13T05:45:26,067 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,067 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,067 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06e18f488ac3dcef5cfa0c667ec59dce: Waiting for close lock at 1731476726067Disabling compacts and flushes for region at 1731476726067Disabling writes for close at 1731476726067Writing region close event to WAL at 1731476726067Closed at 1731476726067 2024-11-13T05:45:26,069 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:45:26,073 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731476726069"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731476726069"}]},"ts":"1731476726069"} 2024-11-13T05:45:26,078 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:45:26,079 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:45:26,082 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476726080"}]},"ts":"1731476726080"} 2024-11-13T05:45:26,086 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T05:45:26,087 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:45:26,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:45:26,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:45:26,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T05:45:26,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:45:26,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:45:26,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T05:45:26,089 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:45:26,089 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:45:26,089 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T05:45:26,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:45:26,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN}] 2024-11-13T05:45:26,093 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN 2024-11-13T05:45:26,094 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,34269,1731476710883; forceNewPlan=false, retain=false 2024-11-13T05:45:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:45:26,247 INFO [bc386b7bf2fe:38153 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T05:45:26,247 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPENING, regionLocation=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:26,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN because future has completed 2024-11-13T05:45:26,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883}] 2024-11-13T05:45:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:45:26,412 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,413 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 06e18f488ac3dcef5cfa0c667ec59dce, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:26,413 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,413 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:26,414 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,414 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,416 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,417 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf1 2024-11-13T05:45:26,418 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:26,418 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:26,418 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,420 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf2 2024-11-13T05:45:26,420 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:26,421 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:26,421 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,422 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,422 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,424 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,424 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,425 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T05:45:26,427 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,430 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:26,431 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 06e18f488ac3dcef5cfa0c667ec59dce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67810964, jitterRate=0.01046210527420044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T05:45:26,431 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,432 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 06e18f488ac3dcef5cfa0c667ec59dce: Running coprocessor pre-open hook at 1731476726414Writing region info on filesystem at 1731476726414Initializing all the Stores at 1731476726415 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476726415Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476726415Cleaning up temporary data from old regions at 1731476726424 (+9 ms)Running coprocessor post-open hooks at 1731476726431 (+7 ms)Region opened successfully at 1731476726432 (+1 ms) 2024-11-13T05:45:26,434 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., pid=6, masterSystemTime=1731476726405 2024-11-13T05:45:26,437 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,437 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,438 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:26,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883 because future has completed 2024-11-13T05:45:26,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:45:26,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883 in 191 msec 2024-11-13T05:45:26,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:45:26,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN in 354 msec 2024-11-13T05:45:26,449 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:45:26,450 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731476726449"}]},"ts":"1731476726449"} 2024-11-13T05:45:26,452 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T05:45:26,453 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:45:26,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 423 msec 2024-11-13T05:45:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:45:26,679 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T05:45:26,679 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T05:45:26,680 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:45:26,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T05:45:26,686 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:45:26,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T05:45:26,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=2] 2024-11-13T05:45:26,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=06e18f488ac3dcef5cfa0c667ec59dce, source=bc386b7bf2fe,34269,1731476710883, destination=bc386b7bf2fe,45573,1731476710694, warming up region on bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:45:26,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=06e18f488ac3dcef5cfa0c667ec59dce, source=bc386b7bf2fe,34269,1731476710883, destination=bc386b7bf2fe,45573,1731476710694, running balancer 2024-11-13T05:45:26,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40811, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:45:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE 2024-11-13T05:45:26,721 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE 2024-11-13T05:45:26,724 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=CLOSING, regionLocation=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:26,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(7855): Warmup {ENCODED => 06e18f488ac3dcef5cfa0c667ec59dce, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:26,726 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,727 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf1 2024-11-13T05:45:26,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE because future has completed 2024-11-13T05:45:26,728 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:26,728 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:45:26,728 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:26,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883}] 2024-11-13T05:45:26,728 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,729 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf2 2024-11-13T05:45:26,730 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:26,730 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(1722): Closing 06e18f488ac3dcef5cfa0c667ec59dce, disabling compactions & flushes 2024-11-13T05:45:26,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. after waiting 0 ms 2024-11-13T05:45:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] regionserver.HRegion(1676): Region close journal for 06e18f488ac3dcef5cfa0c667ec59dce: Waiting for close lock at 1731476726730Disabling compacts and flushes for region at 1731476726730Disabling writes for close at 1731476726731 (+1 ms)Writing region close event to WAL at 1731476726732 (+1 ms)Closed at 1731476726732 2024-11-13T05:45:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T05:45:26,886 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,886 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:45:26,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 06e18f488ac3dcef5cfa0c667ec59dce, disabling compactions & flushes 2024-11-13T05:45:26,887 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. after waiting 0 ms 2024-11-13T05:45:26,887 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,887 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 06e18f488ac3dcef5cfa0c667ec59dce 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T05:45:26,905 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf1/3136ba3f78c24802b608a33c1d64ddde is 35, key is r1/cf1:q/1731476726701/Put/seqid=0 2024-11-13T05:45:26,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741887_1065 (size=4783) 2024-11-13T05:45:26,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741887_1065 (size=4783) 2024-11-13T05:45:26,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741887_1065 (size=4783) 2024-11-13T05:45:26,913 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf1/3136ba3f78c24802b608a33c1d64ddde 2024-11-13T05:45:26,923 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf1/3136ba3f78c24802b608a33c1d64ddde as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde 2024-11-13T05:45:26,930 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T05:45:26,931 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 06e18f488ac3dcef5cfa0c667ec59dce in 44ms, sequenceid=5, compaction requested=false 2024-11-13T05:45:26,931 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T05:45:26,936 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:45:26,939 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:26,939 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 06e18f488ac3dcef5cfa0c667ec59dce: Waiting for close lock at 1731476726887Running coprocessor pre-close hooks at 1731476726887Disabling compacts and flushes for region at 1731476726887Disabling writes for close at 1731476726887Obtaining lock to block concurrent updates at 1731476726887Preparing flush snapshotting stores in 06e18f488ac3dcef5cfa0c667ec59dce at 1731476726887Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731476726888 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. at 1731476726889 (+1 ms)Flushing 06e18f488ac3dcef5cfa0c667ec59dce/cf1: creating writer at 1731476726889Flushing 06e18f488ac3dcef5cfa0c667ec59dce/cf1: appending metadata at 1731476726904 (+15 ms)Flushing 06e18f488ac3dcef5cfa0c667ec59dce/cf1: closing flushed file at 1731476726904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a0de30c: reopening flushed file at 1731476726922 (+18 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 06e18f488ac3dcef5cfa0c667ec59dce in 44ms, sequenceid=5, compaction requested=false at 1731476726931 (+9 ms)Writing region close event to WAL at 1731476726932 (+1 ms)Running coprocessor post-close hooks at 1731476726937 (+5 ms)Closed at 1731476726939 (+2 ms) 2024-11-13T05:45:26,939 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 06e18f488ac3dcef5cfa0c667ec59dce move to bc386b7bf2fe,45573,1731476710694 record at close sequenceid=5 2024-11-13T05:45:26,943 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:26,944 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=CLOSED 2024-11-13T05:45:26,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883 because future has completed 2024-11-13T05:45:26,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T05:45:26,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883 in 221 msec 2024-11-13T05:45:26,953 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,45573,1731476710694; forceNewPlan=false, retain=false 2024-11-13T05:45:27,104 INFO [bc386b7bf2fe:38153 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T05:45:27,105 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPENING, regionLocation=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:27,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE because future has completed 2024-11-13T05:45:27,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694}] 2024-11-13T05:45:27,275 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,276 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 06e18f488ac3dcef5cfa0c667ec59dce, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:27,276 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,276 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:27,276 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,276 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,279 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,280 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf1 2024-11-13T05:45:27,280 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:27,289 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde 2024-11-13T05:45:27,289 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:27,290 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,291 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf2 2024-11-13T05:45:27,291 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:27,291 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:27,292 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,293 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,295 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,295 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,295 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,296 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T05:45:27,298 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,299 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 06e18f488ac3dcef5cfa0c667ec59dce; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70907225, jitterRate=0.05659998953342438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T05:45:27,299 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,299 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 06e18f488ac3dcef5cfa0c667ec59dce: Running coprocessor pre-open hook at 1731476727277Writing region info on filesystem at 1731476727277Initializing all the Stores at 1731476727278 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476727278Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476727278Cleaning up temporary data from old regions at 1731476727296 (+18 ms)Running coprocessor post-open hooks at 1731476727299 (+3 ms)Region opened successfully at 1731476727299 2024-11-13T05:45:27,301 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., pid=9, masterSystemTime=1731476727270 2024-11-13T05:45:27,303 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,303 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,304 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPEN, openSeqNum=9, regionLocation=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:27,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694 because future has completed 2024-11-13T05:45:27,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T05:45:27,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694 in 191 msec 2024-11-13T05:45:27,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE in 591 msec 2024-11-13T05:45:27,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:45:27,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:45:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:56370 deadline: 1731476787333, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=45573 startCode=1731476710694. As of locationSeqNum=5. 2024-11-13T05:45:27,354 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=45573 startCode=1731476710694. As of locationSeqNum=5. 2024-11-13T05:45:27,355 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=45573 startCode=1731476710694. As of locationSeqNum=5. 2024-11-13T05:45:27,355 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,45573,1731476710694, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=45573 startCode=1731476710694. As of locationSeqNum=5. 2024-11-13T05:45:27,467 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:45:27,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:45:27,480 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06e18f488ac3dcef5cfa0c667ec59dce 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T05:45:27,502 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf1/9c5263080b424d07aa88a9c3e7777e2b is 29, key is r1/cf1:/1731476727471/DeleteFamily/seqid=0 2024-11-13T05:45:27,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741888_1066 (size=4906) 2024-11-13T05:45:27,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741888_1066 (size=4906) 2024-11-13T05:45:27,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741888_1066 (size=4906) 2024-11-13T05:45:27,510 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf1/9c5263080b424d07aa88a9c3e7777e2b 2024-11-13T05:45:27,516 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9c5263080b424d07aa88a9c3e7777e2b 2024-11-13T05:45:27,531 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf2/2149759c80c44d28bb00e113211317f8 is 29, key is r1/cf2:/1731476727471/DeleteFamily/seqid=0 2024-11-13T05:45:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741889_1067 (size=4906) 2024-11-13T05:45:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741889_1067 (size=4906) 2024-11-13T05:45:27,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741889_1067 (size=4906) 2024-11-13T05:45:27,538 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf2/2149759c80c44d28bb00e113211317f8 2024-11-13T05:45:27,544 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2149759c80c44d28bb00e113211317f8 2024-11-13T05:45:27,545 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf1/9c5263080b424d07aa88a9c3e7777e2b as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/9c5263080b424d07aa88a9c3e7777e2b 2024-11-13T05:45:27,553 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9c5263080b424d07aa88a9c3e7777e2b 2024-11-13T05:45:27,553 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/9c5263080b424d07aa88a9c3e7777e2b, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T05:45:27,554 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf2/2149759c80c44d28bb00e113211317f8 as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/2149759c80c44d28bb00e113211317f8 2024-11-13T05:45:27,562 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2149759c80c44d28bb00e113211317f8 2024-11-13T05:45:27,562 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/2149759c80c44d28bb00e113211317f8, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T05:45:27,563 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 06e18f488ac3dcef5cfa0c667ec59dce in 84ms, sequenceid=12, compaction requested=false 2024-11-13T05:45:27,563 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 06e18f488ac3dcef5cfa0c667ec59dce: 2024-11-13T05:45:27,566 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T05:45:27,567 DEBUG [Time-limited test {}] regionserver.HStore(1541): 06e18f488ac3dcef5cfa0c667ec59dce/cf1 is initiating major compaction (all files) 2024-11-13T05:45:27,567 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:45:27,567 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:27,568 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 06e18f488ac3dcef5cfa0c667ec59dce/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,568 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde, hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/9c5263080b424d07aa88a9c3e7777e2b] into tmpdir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp, totalSize=9.5 K 2024-11-13T05:45:27,569 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3136ba3f78c24802b608a33c1d64ddde, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731476726701 2024-11-13T05:45:27,570 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9c5263080b424d07aa88a9c3e7777e2b, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T05:45:27,584 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 06e18f488ac3dcef5cfa0c667ec59dce#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:45:27,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741890_1068 (size=4626) 2024-11-13T05:45:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741890_1068 (size=4626) 2024-11-13T05:45:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741890_1068 (size=4626) 2024-11-13T05:45:27,598 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf1/d1ed554bb71f4d28bd13eeb6ba2a5f1a as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/d1ed554bb71f4d28bd13eeb6ba2a5f1a 2024-11-13T05:45:27,613 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 06e18f488ac3dcef5cfa0c667ec59dce/cf1 of 06e18f488ac3dcef5cfa0c667ec59dce into d1ed554bb71f4d28bd13eeb6ba2a5f1a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:45:27,613 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 06e18f488ac3dcef5cfa0c667ec59dce: 2024-11-13T05:45:27,613 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T05:45:27,613 DEBUG [Time-limited test {}] regionserver.HStore(1541): 06e18f488ac3dcef5cfa0c667ec59dce/cf2 is initiating major compaction (all files) 2024-11-13T05:45:27,613 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:45:27,613 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:45:27,613 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 06e18f488ac3dcef5cfa0c667ec59dce/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,613 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/2149759c80c44d28bb00e113211317f8] into tmpdir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp, totalSize=4.8 K 2024-11-13T05:45:27,614 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2149759c80c44d28bb00e113211317f8, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T05:45:27,620 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 06e18f488ac3dcef5cfa0c667ec59dce#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T05:45:27,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741891_1069 (size=4592) 2024-11-13T05:45:27,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741891_1069 (size=4592) 2024-11-13T05:45:27,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741891_1069 (size=4592) 2024-11-13T05:45:27,635 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/.tmp/cf2/d4715daec98042eabe5a67900cdae8b3 as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/d4715daec98042eabe5a67900cdae8b3 2024-11-13T05:45:27,644 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 06e18f488ac3dcef5cfa0c667ec59dce/cf2 of 06e18f488ac3dcef5cfa0c667ec59dce into d4715daec98042eabe5a67900cdae8b3(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:45:27,644 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 06e18f488ac3dcef5cfa0c667ec59dce: 2024-11-13T05:45:27,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=06e18f488ac3dcef5cfa0c667ec59dce, source=bc386b7bf2fe,45573,1731476710694, destination=bc386b7bf2fe,34269,1731476710883, warming up region on bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:27,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=06e18f488ac3dcef5cfa0c667ec59dce, source=bc386b7bf2fe,45573,1731476710694, destination=bc386b7bf2fe,34269,1731476710883, running balancer 2024-11-13T05:45:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE 2024-11-13T05:45:27,650 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE 2024-11-13T05:45:27,652 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=CLOSING, regionLocation=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:27,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(7855): Warmup {ENCODED => 06e18f488ac3dcef5cfa0c667ec59dce, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:27,653 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,655 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf1 2024-11-13T05:45:27,655 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:27,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE because future has completed 2024-11-13T05:45:27,656 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T05:45:27,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694}] 2024-11-13T05:45:27,664 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde 2024-11-13T05:45:27,669 INFO [StoreFileOpener-06e18f488ac3dcef5cfa0c667ec59dce-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9c5263080b424d07aa88a9c3e7777e2b 2024-11-13T05:45:27,669 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/9c5263080b424d07aa88a9c3e7777e2b 2024-11-13T05:45:27,675 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/d1ed554bb71f4d28bd13eeb6ba2a5f1a 2024-11-13T05:45:27,675 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:27,675 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,676 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf2 2024-11-13T05:45:27,676 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:27,683 INFO [StoreFileOpener-06e18f488ac3dcef5cfa0c667ec59dce-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2149759c80c44d28bb00e113211317f8 2024-11-13T05:45:27,683 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/2149759c80c44d28bb00e113211317f8 2024-11-13T05:45:27,689 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/d4715daec98042eabe5a67900cdae8b3 2024-11-13T05:45:27,689 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1722): Closing 06e18f488ac3dcef5cfa0c667ec59dce, disabling compactions & flushes 2024-11-13T05:45:27,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. after waiting 0 ms 2024-11-13T05:45:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1676): Region close journal for 06e18f488ac3dcef5cfa0c667ec59dce: Waiting for close lock at 1731476727690Disabling compacts and flushes for region at 1731476727690Disabling writes for close at 1731476727690Writing region close event to WAL at 1731476727691 (+1 ms)Closed at 1731476727691 2024-11-13T05:45:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T05:45:27,810 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,810 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T05:45:27,810 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 06e18f488ac3dcef5cfa0c667ec59dce, disabling compactions & flushes 2024-11-13T05:45:27,810 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,810 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,810 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. after waiting 0 ms 2024-11-13T05:45:27,810 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,811 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde, hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/9c5263080b424d07aa88a9c3e7777e2b] to archive 2024-11-13T05:45:27,814 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:45:27,818 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/3136ba3f78c24802b608a33c1d64ddde 2024-11-13T05:45:27,821 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/9c5263080b424d07aa88a9c3e7777e2b to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/9c5263080b424d07aa88a9c3e7777e2b 2024-11-13T05:45:27,832 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/2149759c80c44d28bb00e113211317f8] to archive 2024-11-13T05:45:27,833 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:45:27,835 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/2149759c80c44d28bb00e113211317f8 to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/2149759c80c44d28bb00e113211317f8 2024-11-13T05:45:27,840 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T05:45:27,841 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:27,841 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 06e18f488ac3dcef5cfa0c667ec59dce: Waiting for close lock at 1731476727810Running coprocessor pre-close hooks at 1731476727810Disabling compacts and flushes for region at 1731476727810Disabling writes for close at 1731476727810Writing region close event to WAL at 1731476727836 (+26 ms)Running coprocessor post-close hooks at 1731476727841 (+5 ms)Closed at 1731476727841 2024-11-13T05:45:27,841 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 06e18f488ac3dcef5cfa0c667ec59dce move to bc386b7bf2fe,34269,1731476710883 record at close sequenceid=12 2024-11-13T05:45:27,844 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:27,845 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=CLOSED 2024-11-13T05:45:27,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694 because future has completed 2024-11-13T05:45:27,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T05:45:27,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694 in 193 msec 2024-11-13T05:45:27,853 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,34269,1731476710883; forceNewPlan=false, retain=false 2024-11-13T05:45:28,003 INFO [bc386b7bf2fe:38153 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T05:45:28,004 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPENING, regionLocation=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:28,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE because future has completed 2024-11-13T05:45:28,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883}] 2024-11-13T05:45:28,166 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,166 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 06e18f488ac3dcef5cfa0c667ec59dce, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:28,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:28,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,169 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,171 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf1 2024-11-13T05:45:28,171 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:28,178 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/d1ed554bb71f4d28bd13eeb6ba2a5f1a 2024-11-13T05:45:28,178 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:28,178 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,179 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf2 2024-11-13T05:45:28,179 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:28,187 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/d4715daec98042eabe5a67900cdae8b3 2024-11-13T05:45:28,187 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:28,187 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,188 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,189 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,190 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,190 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,191 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T05:45:28,193 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,194 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 06e18f488ac3dcef5cfa0c667ec59dce; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74965595, jitterRate=0.11707441508769989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T05:45:28,194 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,195 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 06e18f488ac3dcef5cfa0c667ec59dce: Running coprocessor pre-open hook at 1731476728167Writing region info on filesystem at 1731476728167Initializing all the Stores at 1731476728169 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476728169Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476728169Cleaning up temporary data from old regions at 1731476728190 (+21 ms)Running coprocessor post-open hooks at 1731476728194 (+4 ms)Region opened successfully at 1731476728195 (+1 ms) 2024-11-13T05:45:28,196 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., pid=12, masterSystemTime=1731476728160 2024-11-13T05:45:28,229 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,230 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,230 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPEN, openSeqNum=18, regionLocation=bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:28,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883 because future has completed 2024-11-13T05:45:28,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T05:45:28,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,34269,1731476710883 in 227 msec 2024-11-13T05:45:28,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, REOPEN/MOVE in 589 msec 2024-11-13T05:45:28,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:45:28,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:45:28,256 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server bc386b7bf2fe,34269,1731476710883: testing ***** 2024-11-13T05:45:28,256 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T05:45:28,258 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T05:45:28,259 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T05:45:28,262 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T05:45:28,263 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T05:45:28,271 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 479417936 }, "NonHeapMemoryUsage": { "committed": 171376640, "init": 7667712, "max": -1, "used": 168654472 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "bc386b7bf2fe", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "bc386b7bf2fe", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60627", "tag.serverName": "bc386b7bf2fe,45573,1731476710694", "tag.clusterId": "ca14ade9-6f32-4196-b83a-8a2c34a32d4c", "tag.Context": "regionserver", "tag.Hostname": "bc386b7bf2fe", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731476710694, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T05:45:28,274 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38153 {}] master.MasterRpcServices(700): bc386b7bf2fe,34269,1731476710883 reported a fatal error: ***** ABORTING region server bc386b7bf2fe,34269,1731476710883: testing ***** 2024-11-13T05:45:28,277 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,34269,1731476710883' ***** 2024-11-13T05:45:28,277 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T05:45:28,278 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:45:28,278 INFO [RS:2;bc386b7bf2fe:34269 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T05:45:28,278 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:45:28,278 INFO [RS:2;bc386b7bf2fe:34269 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T05:45:28,278 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(3091): Received CLOSE for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:56134 deadline: 1731476788278, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=34269 startCode=1731476710883. As of locationSeqNum=12. 2024-11-13T05:45:28,279 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(956): aborting server bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:28,279 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:45:28,279 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06e18f488ac3dcef5cfa0c667ec59dce, disabling compactions & flushes 2024-11-13T05:45:28,279 INFO [RS:2;bc386b7bf2fe:34269 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;bc386b7bf2fe:34269. 2024-11-13T05:45:28,279 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,279 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,279 DEBUG [RS:2;bc386b7bf2fe:34269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:45:28,279 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,45573,1731476710694, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,45573,1731476710694, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=34269 startCode=1731476710883. As of locationSeqNum=12. 2024-11-13T05:45:28,279 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. after waiting 0 ms 2024-11-13T05:45:28,279 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,45573,1731476710694, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=34269 startCode=1731476710883. As of locationSeqNum=12. 2024-11-13T05:45:28,279 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,279 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,45573,1731476710694, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=bc386b7bf2fe port=34269 startCode=1731476710883. As of locationSeqNum=12. 2024-11-13T05:45:28,279 DEBUG [RS:2;bc386b7bf2fe:34269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:28,281 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:45:28,281 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:45:28,281 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:45:28,281 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:45:28,281 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server bc386b7bf2fe,34269,1731476710883 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:28,281 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06e18f488ac3dcef5cfa0c667ec59dce: Waiting for close lock at 1731476728279Running coprocessor pre-close hooks at 1731476728279Disabling compacts and flushes for region at 1731476728279Disabling writes for close at 1731476728279Writing region close event to WAL at 1731476728281 (+2 ms)Running coprocessor post-close hooks at 1731476728281Closed at 1731476728281 2024-11-13T05:45:28,282 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T05:45:28,282 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:28,282 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T05:45:28,282 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:45:28,282 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:45:28,282 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:45:28,282 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:45:28,282 DEBUG [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T05:45:28,282 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:45:28,283 ERROR [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-13T05:45:28,283 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:45:28,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server bc386b7bf2fe,34269,1731476710883 aborting 2024-11-13T05:45:28,283 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:45:28,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server bc386b7bf2fe,34269,1731476710883 aborting 2024-11-13T05:45:28,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 from cache 2024-11-13T05:45:28,283 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476728282Running coprocessor pre-close hooks at 1731476728282Disabling compacts and flushes for region at 1731476728282Disabling writes for close at 1731476728282Writing region close event to WAL at 1731476728283 (+1 ms)Running coprocessor post-close hooks at 1731476728283Closed at 1731476728283 2024-11-13T05:45:28,283 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:45:28,328 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:45:28,329 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:45:28,348 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:45:28,388 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:45:28,389 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1] 2024-11-13T05:45:28,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server bc386b7bf2fe,34269,1731476710883 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:28,391 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server bc386b7bf2fe,34269,1731476710883 aborting 2024-11-13T05:45:28,391 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server bc386b7bf2fe,34269,1731476710883 aborting 2024-11-13T05:45:28,391 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 from cache 2024-11-13T05:45:28,482 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,34269,1731476710883; all regions closed. 2024-11-13T05:45:28,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741836_1012 (size=7323) 2024-11-13T05:45:28,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741836_1012 (size=7323) 2024-11-13T05:45:28,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741836_1012 (size=7323) 2024-11-13T05:45:28,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741835_1011 (size=2288) 2024-11-13T05:45:28,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741835_1011 (size=2288) 2024-11-13T05:45:28,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741835_1011 (size=2288) 2024-11-13T05:45:28,490 DEBUG [RS:2;bc386b7bf2fe:34269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:28,490 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:45:28,490 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:45:28,490 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T05:45:28,491 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:45:28,491 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:45:28,491 INFO [RS:2;bc386b7bf2fe:34269 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34269 2024-11-13T05:45:28,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:45:28,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,34269,1731476710883 2024-11-13T05:45:28,513 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:45:28,521 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,34269,1731476710883] 2024-11-13T05:45:28,529 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,34269,1731476710883 already deleted, retry=false 2024-11-13T05:45:28,529 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of bc386b7bf2fe,34269,1731476710883 on bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:28,534 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bc386b7bf2fe,34269,1731476710883, splitWal=true, meta=true 2024-11-13T05:45:28,537 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for bc386b7bf2fe,34269,1731476710883 (carryingMeta=true) bc386b7bf2fe,34269,1731476710883/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@42cbc2c1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T05:45:28,537 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure bc386b7bf2fe,34269,1731476710883, splitWal=true, meta=true 2024-11-13T05:45:28,538 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,34269,1731476710883, splitWal=true, meta=true, isMeta: true 2024-11-13T05:45:28,540 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting 2024-11-13T05:45:28,541 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,34269,1731476710883 WAL count=1, meta=true 2024-11-13T05:45:28,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta}] 2024-11-13T05:45:28,549 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:28,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, worker=bc386b7bf2fe,46279,1731476710827}] 2024-11-13T05:45:28,597 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:45:28,599 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1] 2024-11-13T05:45:28,602 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:34269 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: bc386b7bf2fe/172.17.0.2:34269 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:28,603 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:34269 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: bc386b7bf2fe/172.17.0.2:34269 2024-11-13T05:45:28,603 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-13T05:45:28,603 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 from cache 2024-11-13T05:45:28,604 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:34269 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: bc386b7bf2fe/172.17.0.2:34269 2024-11-13T05:45:28,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:28,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x101340371ed0003, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:28,622 INFO [RS:2;bc386b7bf2fe:34269 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:45:28,622 INFO [RS:2;bc386b7bf2fe:34269 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,34269,1731476710883; zookeeper connection closed. 2024-11-13T05:45:28,622 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78eb3d34 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78eb3d34 2024-11-13T05:45:28,719 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:45:28,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37555, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:45:28,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46279 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T05:45:28,740 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, size=7.2 K (7323bytes) 2024-11-13T05:45:28,740 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta 2024-11-13T05:45:28,741 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta after 1ms 2024-11-13T05:45:28,767 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:28,767 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta took 27ms 2024-11-13T05:45:28,774 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-13T05:45:28,775 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta so closing down 2024-11-13T05:45:28,775 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:28,776 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta.temp 2024-11-13T05:45:28,802 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta.temp 2024-11-13T05:45:28,802 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:28,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741892_1070 (size=7071) 2024-11-13T05:45:28,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741892_1070 (size=7071) 2024-11-13T05:45:28,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741892_1070 (size=7071) 2024-11-13T05:45:28,810 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:28,812 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta.temp to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T05:45:28,812 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-13T05:45:28,812 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, journal: Splitting hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, size=7.2 K (7323bytes) at 1731476728740Finishing writing output for hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta so closing down at 1731476728775 (+35 ms)Creating recovered edits writer path=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta.temp at 1731476728802 (+27 ms)3 split writer threads finished at 1731476728802Closed recovered edits writer path=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731476728811 (+9 ms)Rename recovered edits hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta.temp to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731476728812 (+1 ms)Processed 16 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1731476728812 2024-11-13T05:45:28,812 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta 2024-11-13T05:45:28,813 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T05:45:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T05:45:28,819 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs 2024-11-13T05:45:28,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T05:45:28,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, worker=bc386b7bf2fe,46279,1731476710827 in 269 msec 2024-11-13T05:45:28,824 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:28,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T05:45:28,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C34269%2C1731476710883.meta.1731476712915.meta, worker=bc386b7bf2fe,46279,1731476710827 in 282 msec 2024-11-13T05:45:28,828 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting dir is empty, no logs to split. 2024-11-13T05:45:28,828 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,34269,1731476710883 WAL count=0, meta=true 2024-11-13T05:45:28,828 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,34269,1731476710883 WAL splitting is done? wals=0, meta=true 2024-11-13T05:45:28,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:45:28,832 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:45:28,834 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:45:28,908 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:45:28,909 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1] 2024-11-13T05:45:28,910 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:34269 this server is in the failed servers list 2024-11-13T05:45:28,910 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:34269 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:28,910 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:28,910 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 from cache 2024-11-13T05:45:28,984 DEBUG [bc386b7bf2fe:38153 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T05:45:28,984 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:45:28,985 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:45:28,985 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:45:28,985 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:45:28,985 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:45:28,985 INFO [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:45:28,985 INFO [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:45:28,985 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:45:28,985 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:28,987 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45573,1731476710694, state=OPENING 2024-11-13T05:45:29,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:29,026 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:29,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:29,026 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:45:29,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:29,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:29,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:29,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45573,1731476710694}] 2024-11-13T05:45:29,187 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:45:29,188 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:29,188 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:45:29,192 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C45573%2C1731476710694.meta, suffix=.meta, logDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,45573,1731476710694, archiveDir=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs, maxLogs=32 2024-11-13T05:45:29,207 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,45573,1731476710694/bc386b7bf2fe%2C45573%2C1731476710694.meta.1731476729192.meta, exclude list is [], retry=0 2024-11-13T05:45:29,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:29,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:29,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:29,235 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,45573,1731476710694/bc386b7bf2fe%2C45573%2C1731476710694.meta.1731476729192.meta 2024-11-13T05:45:29,235 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:29,236 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:29,236 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:45:29,236 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:45:29,236 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:45:29,236 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:45:29,236 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:29,237 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:45:29,237 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:45:29,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:45:29,239 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:45:29,239 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:29,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:29,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:45:29,241 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:45:29,241 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:29,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:29,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:45:29,242 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:45:29,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:29,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:29,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:45:29,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:45:29,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:29,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:45:29,244 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:45:29,245 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740 2024-11-13T05:45:29,247 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740 2024-11-13T05:45:29,248 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T05:45:29,274 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:29,276 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T05:45:29,276 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-13T05:45:29,294 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/info/a8c4231046ee4cb2b89b9c13259bfa5c is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce./info:regioninfo/1731476728230/Put/seqid=0 2024-11-13T05:45:29,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741894_1072 (size=11177) 2024-11-13T05:45:29,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741894_1072 (size=11177) 2024-11-13T05:45:29,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741894_1072 (size=11177) 2024-11-13T05:45:29,302 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/info/a8c4231046ee4cb2b89b9c13259bfa5c 2024-11-13T05:45:29,322 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/ns/ffc040d412fa49b09c0637efa9945eb8 is 43, key is default/ns:d/1731476713118/Put/seqid=0 2024-11-13T05:45:29,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741895_1073 (size=5153) 2024-11-13T05:45:29,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741895_1073 (size=5153) 2024-11-13T05:45:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741895_1073 (size=5153) 2024-11-13T05:45:29,330 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/ns/ffc040d412fa49b09c0637efa9945eb8 2024-11-13T05:45:29,350 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/table/2309363db99b4648bc03c5ba2339ea91 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731476726449/Put/seqid=0 2024-11-13T05:45:29,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741896_1074 (size=5431) 2024-11-13T05:45:29,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741896_1074 (size=5431) 2024-11-13T05:45:29,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741896_1074 (size=5431) 2024-11-13T05:45:29,358 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/table/2309363db99b4648bc03c5ba2339ea91 2024-11-13T05:45:29,365 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/info/a8c4231046ee4cb2b89b9c13259bfa5c as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/info/a8c4231046ee4cb2b89b9c13259bfa5c 2024-11-13T05:45:29,371 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/info/a8c4231046ee4cb2b89b9c13259bfa5c, entries=36, sequenceid=18, filesize=10.9 K 2024-11-13T05:45:29,372 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/ns/ffc040d412fa49b09c0637efa9945eb8 as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/ns/ffc040d412fa49b09c0637efa9945eb8 2024-11-13T05:45:29,378 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/ns/ffc040d412fa49b09c0637efa9945eb8, entries=2, sequenceid=18, filesize=5.0 K 2024-11-13T05:45:29,379 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/table/2309363db99b4648bc03c5ba2339ea91 as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/table/2309363db99b4648bc03c5ba2339ea91 2024-11-13T05:45:29,385 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/table/2309363db99b4648bc03c5ba2339ea91, entries=2, sequenceid=18, filesize=5.3 K 2024-11-13T05:45:29,385 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=18, compaction requested=false; wal=null 2024-11-13T05:45:29,386 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T05:45:29,387 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:45:29,387 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:45:29,388 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:45:29,390 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:45:29,393 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-13T05:45:29,394 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71212565, jitterRate=0.06114991009235382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:45:29,394 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:45:29,395 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731476729237Writing region info on filesystem at 1731476729237Initializing all the Stores at 1731476729238 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476729238Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476729238Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476729238Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731476729238Obtaining lock to block concurrent updates at 1731476729276 (+38 ms)Preparing flush snapshotting stores in 1588230740 at 1731476729276Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731476729276Flushing stores of hbase:meta,,1.1588230740 at 1731476729276Flushing 1588230740/info: creating writer at 1731476729277 (+1 ms)Flushing 1588230740/info: appending metadata at 1731476729294 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731476729294Flushing 1588230740/ns: creating writer at 1731476729308 (+14 ms)Flushing 1588230740/ns: appending metadata at 1731476729322 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731476729322Flushing 1588230740/table: creating writer at 1731476729336 (+14 ms)Flushing 1588230740/table: appending metadata at 1731476729350 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731476729350Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e1315ae: reopening flushed file at 1731476729364 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55351c41: reopening flushed file at 1731476729371 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27866877: reopening flushed file at 1731476729378 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=18, compaction requested=false; wal=null at 1731476729385 (+7 ms)Cleaning up temporary data from old regions at 1731476729387 (+2 ms)Running coprocessor post-open hooks at 1731476729394 (+7 ms)Region opened successfully at 1731476729395 (+1 ms) 2024-11-13T05:45:29,396 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731476729178 2024-11-13T05:45:29,398 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:45:29,398 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:45:29,399 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:29,400 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45573,1731476710694, state=OPEN 2024-11-13T05:45:29,418 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:45:29,419 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45573,1731476710694, seqNum=-1] 2024-11-13T05:45:29,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:29,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:29,426 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:45:29,426 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:29,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:29,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:29,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:45:29,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T05:45:29,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45573,1731476710694 in 400 msec 2024-11-13T05:45:29,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T05:45:29,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 598 msec 2024-11-13T05:45:29,431 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): bc386b7bf2fe,34269,1731476710883 had 2 regions 2024-11-13T05:45:29,432 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,34269,1731476710883, splitWal=true, meta=true, isMeta: false 2024-11-13T05:45:29,434 INFO [PEWorker-2 {}] master.SplitWALManager(105): bc386b7bf2fe,34269,1731476710883 WAL count=1, meta=false 2024-11-13T05:45:29,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C34269%2C1731476710883.1731476712488}] 2024-11-13T05:45:29,437 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:29,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, worker=bc386b7bf2fe,46279,1731476710827}] 2024-11-13T05:45:29,537 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:34269 this server is in the failed servers list 2024-11-13T05:45:29,538 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:34269 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:29,538 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:29,538 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=12 from cache 2024-11-13T05:45:29,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46279 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-13T05:45:29,612 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, size=2.2 K (2288bytes) 2024-11-13T05:45:29,612 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 2024-11-13T05:45:29,613 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 after 0ms 2024-11-13T05:45:29,640 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:29,640 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 took 28ms 2024-11-13T05:45:29,643 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 06e18f488ac3dcef5cfa0c667ec59dce: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T05:45:29,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 so closing down 2024-11-13T05:45:29,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:29,644 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:29,644 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-13T05:45:29,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, journal: Splitting hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, size=2.2 K (2288bytes) at 1731476729612Finishing writing output for hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 so closing down at 1731476729644 (+32 ms)3 split writer threads finished at 1731476729644Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731476729644 2024-11-13T05:45:29,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 2024-11-13T05:45:29,644 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-13T05:45:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38153 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-13T05:45:29,649 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting/bc386b7bf2fe%2C34269%2C1731476710883.1731476712488 to hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs 2024-11-13T05:45:29,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-13T05:45:29,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, worker=bc386b7bf2fe,46279,1731476710827 in 211 msec 2024-11-13T05:45:29,653 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:29,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-13T05:45:29,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C34269%2C1731476710883.1731476712488, worker=bc386b7bf2fe,46279,1731476710827 in 219 msec 2024-11-13T05:45:29,658 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/WALs/bc386b7bf2fe,34269,1731476710883-splitting dir is empty, no logs to split. 2024-11-13T05:45:29,658 INFO [PEWorker-2 {}] master.SplitWALManager(105): bc386b7bf2fe,34269,1731476710883 WAL count=0, meta=false 2024-11-13T05:45:29,658 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,34269,1731476710883 WAL splitting is done? wals=0, meta=false 2024-11-13T05:45:29,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN}] 2024-11-13T05:45:29,662 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN 2024-11-13T05:45:29,663 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:45:29,757 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18] 2024-11-13T05:45:29,758 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:34269 this server is in the failed servers list 2024-11-13T05:45:29,759 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:34269 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:29,759 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:29,759 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18 from cache 2024-11-13T05:45:29,814 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:45:29,814 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:45:29,814 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:45:29,814 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:45:29,814 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:45:29,814 INFO [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:45:29,814 INFO [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:45:29,814 DEBUG [bc386b7bf2fe:38153 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:45:29,815 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPENING, regionLocation=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:29,816 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:34269 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:34269 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:29,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:34269 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:34269 2024-11-13T05:45:29,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:45:29,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,34269,1731476710883, seqNum=-1 from cache 2024-11-13T05:45:29,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:34269 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:34269 2024-11-13T05:45:29,927 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:45:29,928 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45573,1731476710694, seqNum=-1] 2024-11-13T05:45:29,928 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:45:29,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:45:29,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN because future has completed 2024-11-13T05:45:29,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694}] 2024-11-13T05:45:30,074 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18] 2024-11-13T05:45:30,074 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:34269 this server is in the failed servers list 2024-11-13T05:45:30,075 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:34269 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:30,075 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:34269 2024-11-13T05:45:30,075 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,34269,1731476710883, seqNum=18 from cache 2024-11-13T05:45:30,088 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:30,089 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 06e18f488ac3dcef5cfa0c667ec59dce, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:30,089 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,089 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:30,089 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,089 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,091 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,092 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf1 2024-11-13T05:45:30,092 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,100 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf1/d1ed554bb71f4d28bd13eeb6ba2a5f1a 2024-11-13T05:45:30,100 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,100 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,101 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06e18f488ac3dcef5cfa0c667ec59dce columnFamilyName cf2 2024-11-13T05:45:30,102 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,111 DEBUG [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/cf2/d4715daec98042eabe5a67900cdae8b3 2024-11-13T05:45:30,111 INFO [StoreOpener-06e18f488ac3dcef5cfa0c667ec59dce-1 {}] regionserver.HStore(327): Store=06e18f488ac3dcef5cfa0c667ec59dce/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,111 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,112 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,113 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,114 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,114 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,115 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T05:45:30,117 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,117 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 06e18f488ac3dcef5cfa0c667ec59dce; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63600582, jitterRate=-0.05227747559547424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T05:45:30,118 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:30,118 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 06e18f488ac3dcef5cfa0c667ec59dce: Running coprocessor pre-open hook at 1731476730089Writing region info on filesystem at 1731476730089Initializing all the Stores at 1731476730090 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730090Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730091 (+1 ms)Cleaning up temporary data from old regions at 1731476730114 (+23 ms)Running coprocessor post-open hooks at 1731476730118 (+4 ms)Region opened successfully at 1731476730118 2024-11-13T05:45:30,120 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., pid=21, masterSystemTime=1731476730085 2024-11-13T05:45:30,122 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:30,122 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:30,123 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=06e18f488ac3dcef5cfa0c667ec59dce, regionState=OPEN, openSeqNum=18, regionLocation=bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:30,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694 because future has completed 2024-11-13T05:45:30,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T05:45:30,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 06e18f488ac3dcef5cfa0c667ec59dce, server=bc386b7bf2fe,45573,1731476710694 in 195 msec 2024-11-13T05:45:30,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-13T05:45:30,133 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server bc386b7bf2fe,34269,1731476710883 after splitting done 2024-11-13T05:45:30,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06e18f488ac3dcef5cfa0c667ec59dce, ASSIGN in 470 msec 2024-11-13T05:45:30,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,34269,1731476710883, splitWal=true, meta=true in 1.6030 sec 2024-11-13T05:45:30,286 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T05:45:30,286 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:30,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T05:45:30,287 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:30,289 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T05:45:30,289 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:30,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., hostname=bc386b7bf2fe,45573,1731476710694, seqNum=18] 2024-11-13T05:45:30,610 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 408), OpenFileDescriptor=1046 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=324 (was 327), ProcessCount=11 (was 11), AvailableMemoryMB=7414 (was 7475) 2024-11-13T05:45:30,612 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-11-13T05:45:30,625 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1046, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=7414 2024-11-13T05:45:30,626 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-11-13T05:45:30,644 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:30,645 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:30,646 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:30,649 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-49533445, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-49533445, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:30,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-49533445/hregion-49533445.1731476730649, exclude list is [], retry=0 2024-11-13T05:45:30,663 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:30,663 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:30,664 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:30,689 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-49533445/hregion-49533445.1731476730649 2024-11-13T05:45:30,689 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:30,690 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2994ea58c6ad3085f5643ed7695c6c9c, NAME => 'testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:30,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741898_1076 (size=67) 2024-11-13T05:45:30,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741898_1076 (size=67) 2024-11-13T05:45:30,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741898_1076 (size=67) 2024-11-13T05:45:30,699 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:30,700 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,701 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName a 2024-11-13T05:45:30,701 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,702 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,702 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,703 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName b 2024-11-13T05:45:30,704 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,704 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,704 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,705 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName c 2024-11-13T05:45:30,705 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,706 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,706 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,706 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,707 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,708 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,708 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,708 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:30,709 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,712 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:30,712 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2994ea58c6ad3085f5643ed7695c6c9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69496474, jitterRate=0.0355781614780426}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:30,713 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2994ea58c6ad3085f5643ed7695c6c9c: Writing region info on filesystem at 1731476730699Initializing all the Stores at 1731476730700 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730700Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730700Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730700Cleaning up temporary data from old regions at 1731476730708 (+8 ms)Region opened successfully at 1731476730713 (+5 ms) 2024-11-13T05:45:30,713 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2994ea58c6ad3085f5643ed7695c6c9c, disabling compactions & flushes 2024-11-13T05:45:30,713 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:30,713 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:30,713 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. after waiting 0 ms 2024-11-13T05:45:30,713 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:30,713 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:30,713 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2994ea58c6ad3085f5643ed7695c6c9c: Waiting for close lock at 1731476730713Disabling compacts and flushes for region at 1731476730713Disabling writes for close at 1731476730713Writing region close event to WAL at 1731476730713Closed at 1731476730713 2024-11-13T05:45:30,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741897_1075 (size=175) 2024-11-13T05:45:30,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741897_1075 (size=175) 2024-11-13T05:45:30,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741897_1075 (size=175) 2024-11-13T05:45:30,716 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-49533445/hregion-49533445.1731476730649 not finished, retry = 0 2024-11-13T05:45:30,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:30,821 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-49533445:(num 1731476730649) 2024-11-13T05:45:30,822 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:30,826 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:30,842 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826, exclude list is [], retry=0 2024-11-13T05:45:30,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:30,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:30,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:30,871 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826 2024-11-13T05:45:30,871 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:30,871 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2994ea58c6ad3085f5643ed7695c6c9c, NAME => 'testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:30,872 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:30,872 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,872 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,873 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,874 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName a 2024-11-13T05:45:30,874 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,874 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,874 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,875 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName b 2024-11-13T05:45:30,875 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,875 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,876 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,876 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName c 2024-11-13T05:45:30,876 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:30,877 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:30,877 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,877 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,879 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,879 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,879 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:30,880 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:30,881 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2994ea58c6ad3085f5643ed7695c6c9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70145022, jitterRate=0.04524227976799011}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:30,881 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2994ea58c6ad3085f5643ed7695c6c9c: Writing region info on filesystem at 1731476730872Initializing all the Stores at 1731476730873 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730873Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730873Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476730873Cleaning up temporary data from old regions at 1731476730879 (+6 ms)Region opened successfully at 1731476730881 (+2 ms) 2024-11-13T05:45:30,906 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2994ea58c6ad3085f5643ed7695c6c9c 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T05:45:30,922 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/a/2ddf0dbe20284a19b4d02f2dd6f467d0 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731476730881/Put/seqid=0 2024-11-13T05:45:30,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741900_1078 (size=5958) 2024-11-13T05:45:30,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741900_1078 (size=5958) 2024-11-13T05:45:30,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741900_1078 (size=5958) 2024-11-13T05:45:30,929 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/a/2ddf0dbe20284a19b4d02f2dd6f467d0 2024-11-13T05:45:30,948 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/b/ef0f0b8984f94e10aeafcfdfa71896b3 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731476730890/Put/seqid=0 2024-11-13T05:45:30,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741901_1079 (size=5958) 2024-11-13T05:45:30,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741901_1079 (size=5958) 2024-11-13T05:45:30,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741901_1079 (size=5958) 2024-11-13T05:45:30,955 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/b/ef0f0b8984f94e10aeafcfdfa71896b3 2024-11-13T05:45:30,973 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/c/726ff2595b48442cb5f51eff8214d629 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731476730897/Put/seqid=0 2024-11-13T05:45:30,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741902_1080 (size=5958) 2024-11-13T05:45:30,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741902_1080 (size=5958) 2024-11-13T05:45:30,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741902_1080 (size=5958) 2024-11-13T05:45:30,981 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/c/726ff2595b48442cb5f51eff8214d629 2024-11-13T05:45:30,986 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/a/2ddf0dbe20284a19b4d02f2dd6f467d0 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/a/2ddf0dbe20284a19b4d02f2dd6f467d0 2024-11-13T05:45:30,991 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/a/2ddf0dbe20284a19b4d02f2dd6f467d0, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T05:45:30,992 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/b/ef0f0b8984f94e10aeafcfdfa71896b3 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/b/ef0f0b8984f94e10aeafcfdfa71896b3 2024-11-13T05:45:30,998 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/b/ef0f0b8984f94e10aeafcfdfa71896b3, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T05:45:31,000 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/c/726ff2595b48442cb5f51eff8214d629 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/c/726ff2595b48442cb5f51eff8214d629 2024-11-13T05:45:31,006 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/c/726ff2595b48442cb5f51eff8214d629, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T05:45:31,007 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2994ea58c6ad3085f5643ed7695c6c9c in 102ms, sequenceid=33, compaction requested=false 2024-11-13T05:45:31,007 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2994ea58c6ad3085f5643ed7695c6c9c: 2024-11-13T05:45:31,007 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2994ea58c6ad3085f5643ed7695c6c9c, disabling compactions & flushes 2024-11-13T05:45:31,007 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:31,007 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:31,007 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. after waiting 0 ms 2024-11-13T05:45:31,007 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:31,009 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. 2024-11-13T05:45:31,009 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2994ea58c6ad3085f5643ed7695c6c9c: Waiting for close lock at 1731476731007Disabling compacts and flushes for region at 1731476731007Disabling writes for close at 1731476731007Writing region close event to WAL at 1731476731009 (+2 ms)Closed at 1731476731009 2024-11-13T05:45:31,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741899_1077 (size=6311) 2024-11-13T05:45:31,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741899_1077 (size=6311) 2024-11-13T05:45:31,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741899_1077 (size=6311) 2024-11-13T05:45:31,016 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/b/ef0f0b8984f94e10aeafcfdfa71896b3 to hdfs://localhost:39659/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/b/ef0f0b8984f94e10aeafcfdfa71896b3 2024-11-13T05:45:31,032 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826, size=6.2 K (6311bytes) 2024-11-13T05:45:31,032 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826 2024-11-13T05:45:31,032 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826 after 0ms 2024-11-13T05:45:31,068 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:31,068 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826 took 37ms 2024-11-13T05:45:31,070 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826 so closing down 2024-11-13T05:45:31,071 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:31,072 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731476730826.temp 2024-11-13T05:45:31,099 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000003-wal.1731476730826.temp 2024-11-13T05:45:31,100 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:31,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741903_1081 (size=5575) 2024-11-13T05:45:31,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741903_1081 (size=5575) 2024-11-13T05:45:31,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741903_1081 (size=5575) 2024-11-13T05:45:31,111 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000003-wal.1731476730826.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:31,112 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000003-wal.1731476730826.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000032 2024-11-13T05:45:31,112 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 44 ms; skipped=2; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-13T05:45:31,113 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826, journal: Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826, size=6.2 K (6311bytes) at 1731476731032Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826 so closing down at 1731476731071 (+39 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000003-wal.1731476730826.temp at 1731476731100 (+29 ms)3 split writer threads finished at 1731476731100Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000003-wal.1731476730826.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731476731111 (+11 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000003-wal.1731476730826.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000032 at 1731476731112 (+1 ms)Processed 32 edits across 1 Regions in 44 ms; skipped=2; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731476731112 2024-11-13T05:45:31,114 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476730826 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476730826 2024-11-13T05:45:31,115 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000032 2024-11-13T05:45:31,115 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:31,117 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:31,136 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476731118, exclude list is [], retry=0 2024-11-13T05:45:31,139 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:31,140 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:31,140 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:31,170 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476731118 2024-11-13T05:45:31,170 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:31,170 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2994ea58c6ad3085f5643ed7695c6c9c, NAME => 'testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:31,170 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:31,171 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,171 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,172 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,173 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName a 2024-11-13T05:45:31,173 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,179 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/a/2ddf0dbe20284a19b4d02f2dd6f467d0 2024-11-13T05:45:31,180 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,180 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,181 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName b 2024-11-13T05:45:31,181 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,182 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,182 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,182 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2994ea58c6ad3085f5643ed7695c6c9c columnFamilyName c 2024-11-13T05:45:31,183 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,188 DEBUG [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/c/726ff2595b48442cb5f51eff8214d629 2024-11-13T05:45:31,188 INFO [StoreOpener-2994ea58c6ad3085f5643ed7695c6c9c-1 {}] regionserver.HStore(327): Store=2994ea58c6ad3085f5643ed7695c6c9c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,188 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,189 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,191 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,191 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000032 2024-11-13T05:45:31,219 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:31,220 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000032 2024-11-13T05:45:31,220 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2994ea58c6ad3085f5643ed7695c6c9c 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T05:45:31,234 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/b/4a7ae09c89564d9eb6b1612b2fc1b1df is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731476730890/Put/seqid=0 2024-11-13T05:45:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741905_1083 (size=5958) 2024-11-13T05:45:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741905_1083 (size=5958) 2024-11-13T05:45:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741905_1083 (size=5958) 2024-11-13T05:45:31,241 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/b/4a7ae09c89564d9eb6b1612b2fc1b1df 2024-11-13T05:45:31,246 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/.tmp/b/4a7ae09c89564d9eb6b1612b2fc1b1df as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/b/4a7ae09c89564d9eb6b1612b2fc1b1df 2024-11-13T05:45:31,251 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/b/4a7ae09c89564d9eb6b1612b2fc1b1df, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T05:45:31,251 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2994ea58c6ad3085f5643ed7695c6c9c in 31ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T05:45:31,252 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/0000000000000000032 2024-11-13T05:45:31,253 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,253 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,253 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:31,255 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2994ea58c6ad3085f5643ed7695c6c9c 2024-11-13T05:45:31,257 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/2994ea58c6ad3085f5643ed7695c6c9c/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T05:45:31,258 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2994ea58c6ad3085f5643ed7695c6c9c; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73099740, jitterRate=0.08927100896835327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:31,258 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2994ea58c6ad3085f5643ed7695c6c9c: Writing region info on filesystem at 1731476731171Initializing all the Stores at 1731476731172 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731172Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731172Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731172Obtaining lock to block concurrent updates at 1731476731220 (+48 ms)Preparing flush snapshotting stores in 2994ea58c6ad3085f5643ed7695c6c9c at 1731476731220Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731476731221 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731476730644.2994ea58c6ad3085f5643ed7695c6c9c. at 1731476731221Flushing 2994ea58c6ad3085f5643ed7695c6c9c/b: creating writer at 1731476731221Flushing 2994ea58c6ad3085f5643ed7695c6c9c/b: appending metadata at 1731476731234 (+13 ms)Flushing 2994ea58c6ad3085f5643ed7695c6c9c/b: closing flushed file at 1731476731234Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26255dd4: reopening flushed file at 1731476731245 (+11 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2994ea58c6ad3085f5643ed7695c6c9c in 31ms, sequenceid=32, compaction requested=false; wal=null at 1731476731251 (+6 ms)Cleaning up temporary data from old regions at 1731476731253 (+2 ms)Region opened successfully at 1731476731258 (+5 ms) 2024-11-13T05:45:31,279 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414 (was 405) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:54904 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:54718 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:44558 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:39028 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:38852 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:44354 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1122 (was 1046) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=324 (was 324), ProcessCount=11 (was 11), AvailableMemoryMB=7404 (was 7414) 2024-11-13T05:45:31,280 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-11-13T05:45:31,294 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1122, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=7404 2024-11-13T05:45:31,294 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-11-13T05:45:31,313 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:31,315 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:31,316 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:31,318 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89926667, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-89926667, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:31,330 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89926667/hregion-89926667.1731476731319, exclude list is [], retry=0 2024-11-13T05:45:31,333 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:31,333 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:31,333 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:31,357 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89926667/hregion-89926667.1731476731319 2024-11-13T05:45:31,357 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:31,357 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 690182eda9dc1735195eaa347a9ec8ff, NAME => 'testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:31,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741907_1085 (size=68) 2024-11-13T05:45:31,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741907_1085 (size=68) 2024-11-13T05:45:31,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741907_1085 (size=68) 2024-11-13T05:45:31,366 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:31,368 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,369 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName a 2024-11-13T05:45:31,369 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,370 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,370 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,371 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName b 2024-11-13T05:45:31,372 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,372 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,372 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,374 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName c 2024-11-13T05:45:31,374 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,374 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,374 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,376 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,377 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,377 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,378 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:31,379 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,381 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:31,381 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690182eda9dc1735195eaa347a9ec8ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65405669, jitterRate=-0.02537958323955536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:31,382 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690182eda9dc1735195eaa347a9ec8ff: Writing region info on filesystem at 1731476731366Initializing all the Stores at 1731476731367 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731367Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731367Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731367Cleaning up temporary data from old regions at 1731476731377 (+10 ms)Region opened successfully at 1731476731382 (+5 ms) 2024-11-13T05:45:31,382 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 690182eda9dc1735195eaa347a9ec8ff, disabling compactions & flushes 2024-11-13T05:45:31,382 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:31,382 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:31,382 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. after waiting 0 ms 2024-11-13T05:45:31,382 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:31,382 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:31,382 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 690182eda9dc1735195eaa347a9ec8ff: Waiting for close lock at 1731476731382Disabling compacts and flushes for region at 1731476731382Disabling writes for close at 1731476731382Writing region close event to WAL at 1731476731382Closed at 1731476731382 2024-11-13T05:45:31,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741906_1084 (size=175) 2024-11-13T05:45:31,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741906_1084 (size=175) 2024-11-13T05:45:31,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741906_1084 (size=175) 2024-11-13T05:45:31,387 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:31,387 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89926667:(num 1731476731319) 2024-11-13T05:45:31,387 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:31,389 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:31,401 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389, exclude list is [], retry=0 2024-11-13T05:45:31,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:31,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:31,405 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:31,431 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389 2024-11-13T05:45:31,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:31,503 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 690182eda9dc1735195eaa347a9ec8ff, NAME => 'testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:31,505 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,505 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:31,506 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,506 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,508 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,509 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName a 2024-11-13T05:45:31,510 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,510 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,511 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,512 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName b 2024-11-13T05:45:31,512 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,512 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,513 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,514 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName c 2024-11-13T05:45:31,514 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:31,514 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:31,514 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,515 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,516 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,517 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,517 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,518 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:31,520 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,520 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690182eda9dc1735195eaa347a9ec8ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74490441, jitterRate=0.1099940687417984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:31,521 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:31,521 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690182eda9dc1735195eaa347a9ec8ff: Running coprocessor pre-open hook at 1731476731506Writing region info on filesystem at 1731476731506Initializing all the Stores at 1731476731507 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731507Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731507Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476731508 (+1 ms)Cleaning up temporary data from old regions at 1731476731517 (+9 ms)Running coprocessor post-open hooks at 1731476731521 (+4 ms)Region opened successfully at 1731476731521 2024-11-13T05:45:31,535 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 690182eda9dc1735195eaa347a9ec8ff 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T05:45:31,536 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:31,954 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:45:32,537 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:33,539 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:34,541 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:35,542 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:36,544 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:37,012 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:45:37,545 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:38,547 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:39,108 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:45:39,548 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:40,286 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T05:45:40,286 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:40,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T05:45:40,287 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:40,549 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 690182eda9dc1735195eaa347a9ec8ff/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:40,552 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 690182eda9dc1735195eaa347a9ec8ff: 2024-11-13T05:45:40,552 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:40,567 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 690182eda9dc1735195eaa347a9ec8ff: 2024-11-13T05:45:40,567 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T05:45:40,567 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 690182eda9dc1735195eaa347a9ec8ff, disabling compactions & flushes 2024-11-13T05:45:40,567 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:40,567 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:40,567 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. after waiting 0 ms 2024-11-13T05:45:40,567 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:40,567 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:40,567 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. 2024-11-13T05:45:40,568 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 690182eda9dc1735195eaa347a9ec8ff: Waiting for close lock at 1731476740567Running coprocessor pre-close hooks at 1731476740567Disabling compacts and flushes for region at 1731476740567Disabling writes for close at 1731476740567Writing region close event to WAL at 1731476740567Running coprocessor post-close hooks at 1731476740567Closed at 1731476740567 2024-11-13T05:45:40,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741908_1086 (size=4236) 2024-11-13T05:45:40,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741908_1086 (size=4236) 2024-11-13T05:45:40,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741908_1086 (size=4236) 2024-11-13T05:45:40,585 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389, size=4.1 K (4236bytes) 2024-11-13T05:45:40,586 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389 2024-11-13T05:45:40,586 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389 after 0ms 2024-11-13T05:45:40,612 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:40,613 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389 took 28ms 2024-11-13T05:45:40,615 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389 so closing down 2024-11-13T05:45:40,615 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:40,616 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731476731389.temp 2024-11-13T05:45:40,641 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000004-wal.1731476731389.temp 2024-11-13T05:45:40,642 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:40,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741909_1087 (size=3245) 2024-11-13T05:45:40,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741909_1087 (size=3245) 2024-11-13T05:45:40,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741909_1087 (size=3245) 2024-11-13T05:45:40,650 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000004-wal.1731476731389.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:40,651 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000004-wal.1731476731389.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000026 2024-11-13T05:45:40,651 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-13T05:45:40,651 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389, journal: Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389, size=4.1 K (4236bytes) at 1731476740585Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389 so closing down at 1731476740615 (+30 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000004-wal.1731476731389.temp at 1731476740641 (+26 ms)3 split writer threads finished at 1731476740642 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000004-wal.1731476731389.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731476740650 (+8 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000004-wal.1731476731389.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000026 at 1731476740651 (+1 ms)Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731476740651 2024-11-13T05:45:40,653 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476731389 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476731389 2024-11-13T05:45:40,654 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000026 2024-11-13T05:45:40,654 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:40,656 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:40,669 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476740656, exclude list is [], retry=0 2024-11-13T05:45:40,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:40,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:40,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:40,698 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476740656 2024-11-13T05:45:40,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:40,699 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 690182eda9dc1735195eaa347a9ec8ff, NAME => 'testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:40,699 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,699 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:40,700 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,700 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,702 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,703 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName a 2024-11-13T05:45:40,703 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:40,703 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:40,704 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,704 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName b 2024-11-13T05:45:40,705 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:40,705 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:40,705 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,706 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 690182eda9dc1735195eaa347a9ec8ff columnFamilyName c 2024-11-13T05:45:40,706 DEBUG [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:40,707 INFO [StoreOpener-690182eda9dc1735195eaa347a9ec8ff-1 {}] regionserver.HStore(327): Store=690182eda9dc1735195eaa347a9ec8ff/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:40,707 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,708 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,709 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000026 2024-11-13T05:45:40,767 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:40,769 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000026 2024-11-13T05:45:40,770 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 690182eda9dc1735195eaa347a9ec8ff 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T05:45:40,791 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/a/8ba546cb3ce14429afc42f5a762e95c7 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731476740560/Put/seqid=0 2024-11-13T05:45:40,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741911_1089 (size=5523) 2024-11-13T05:45:40,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741911_1089 (size=5523) 2024-11-13T05:45:40,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741911_1089 (size=5523) 2024-11-13T05:45:40,798 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/a/8ba546cb3ce14429afc42f5a762e95c7 2024-11-13T05:45:40,818 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/b/567bb666c2fc49b4bccd4a12d906a9e5 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731476740553/Put/seqid=0 2024-11-13T05:45:40,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741912_1090 (size=5524) 2024-11-13T05:45:40,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741912_1090 (size=5524) 2024-11-13T05:45:40,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741912_1090 (size=5524) 2024-11-13T05:45:40,824 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/b/567bb666c2fc49b4bccd4a12d906a9e5 2024-11-13T05:45:40,844 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/c/1d98cb4926824f579f4b623ce77ba31b is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731476740557/Put/seqid=0 2024-11-13T05:45:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741913_1091 (size=5457) 2024-11-13T05:45:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741913_1091 (size=5457) 2024-11-13T05:45:40,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741913_1091 (size=5457) 2024-11-13T05:45:40,851 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/c/1d98cb4926824f579f4b623ce77ba31b 2024-11-13T05:45:40,857 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/a/8ba546cb3ce14429afc42f5a762e95c7 as hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/a/8ba546cb3ce14429afc42f5a762e95c7 2024-11-13T05:45:40,862 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/a/8ba546cb3ce14429afc42f5a762e95c7, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T05:45:40,863 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/b/567bb666c2fc49b4bccd4a12d906a9e5 as hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/b/567bb666c2fc49b4bccd4a12d906a9e5 2024-11-13T05:45:40,868 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/b/567bb666c2fc49b4bccd4a12d906a9e5, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T05:45:40,869 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/.tmp/c/1d98cb4926824f579f4b623ce77ba31b as hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/c/1d98cb4926824f579f4b623ce77ba31b 2024-11-13T05:45:40,874 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/c/1d98cb4926824f579f4b623ce77ba31b, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T05:45:40,874 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 690182eda9dc1735195eaa347a9ec8ff in 104ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T05:45:40,875 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/0000000000000000026 2024-11-13T05:45:40,876 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,876 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,877 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:40,878 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,880 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsAfterAbortingFlush/690182eda9dc1735195eaa347a9ec8ff/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T05:45:40,881 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 690182eda9dc1735195eaa347a9ec8ff; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62170934, jitterRate=-0.07358089089393616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:40,881 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 690182eda9dc1735195eaa347a9ec8ff 2024-11-13T05:45:40,882 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 690182eda9dc1735195eaa347a9ec8ff: Running coprocessor pre-open hook at 1731476740700Writing region info on filesystem at 1731476740700Initializing all the Stores at 1731476740701 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476740701Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476740701Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476740701Obtaining lock to block concurrent updates at 1731476740770 (+69 ms)Preparing flush snapshotting stores in 690182eda9dc1735195eaa347a9ec8ff at 1731476740770Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731476740770Flushing stores of testReplayEditsAfterAbortingFlush,,1731476731314.690182eda9dc1735195eaa347a9ec8ff. at 1731476740770Flushing 690182eda9dc1735195eaa347a9ec8ff/a: creating writer at 1731476740770Flushing 690182eda9dc1735195eaa347a9ec8ff/a: appending metadata at 1731476740790 (+20 ms)Flushing 690182eda9dc1735195eaa347a9ec8ff/a: closing flushed file at 1731476740790Flushing 690182eda9dc1735195eaa347a9ec8ff/b: creating writer at 1731476740804 (+14 ms)Flushing 690182eda9dc1735195eaa347a9ec8ff/b: appending metadata at 1731476740817 (+13 ms)Flushing 690182eda9dc1735195eaa347a9ec8ff/b: closing flushed file at 1731476740817Flushing 690182eda9dc1735195eaa347a9ec8ff/c: creating writer at 1731476740830 (+13 ms)Flushing 690182eda9dc1735195eaa347a9ec8ff/c: appending metadata at 1731476740844 (+14 ms)Flushing 690182eda9dc1735195eaa347a9ec8ff/c: closing flushed file at 1731476740844Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@eda72e0: reopening flushed file at 1731476740856 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c89b8b0: reopening flushed file at 1731476740862 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53a200a4: reopening flushed file at 1731476740869 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 690182eda9dc1735195eaa347a9ec8ff in 104ms, sequenceid=26, compaction requested=false; wal=null at 1731476740874 (+5 ms)Cleaning up temporary data from old regions at 1731476740876 (+2 ms)Running coprocessor post-open hooks at 1731476740881 (+5 ms)Region opened successfully at 1731476740882 (+1 ms) 2024-11-13T05:45:40,903 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 414) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:32780 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:48186 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:58812 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:58826 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:60970 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:48196 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1180 (was 1122) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 324), ProcessCount=11 (was 11), AvailableMemoryMB=7384 (was 7404) 2024-11-13T05:45:40,903 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-13T05:45:40,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=7383 2024-11-13T05:45:40,915 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-13T05:45:40,928 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:40,930 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:40,930 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:40,932 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77232107, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-77232107, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:40,944 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77232107/hregion-77232107.1731476740933, exclude list is [], retry=0 2024-11-13T05:45:40,946 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:40,946 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:40,946 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:40,971 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77232107/hregion-77232107.1731476740933 2024-11-13T05:45:40,971 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:40,971 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c7b01a6d6207b5f0098648c3161dd6a6, NAME => 'testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:40,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741915_1093 (size=61) 2024-11-13T05:45:40,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741915_1093 (size=61) 2024-11-13T05:45:40,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741915_1093 (size=61) 2024-11-13T05:45:40,985 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:40,986 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:40,988 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7b01a6d6207b5f0098648c3161dd6a6 columnFamilyName a 2024-11-13T05:45:40,988 DEBUG [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:40,988 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(327): Store=c7b01a6d6207b5f0098648c3161dd6a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:40,989 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:40,989 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:40,990 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:40,990 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:40,990 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:40,992 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:40,994 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:40,995 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c7b01a6d6207b5f0098648c3161dd6a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64422438, jitterRate=-0.040030866861343384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:40,995 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c7b01a6d6207b5f0098648c3161dd6a6: Writing region info on filesystem at 1731476740985Initializing all the Stores at 1731476740986 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476740986Cleaning up temporary data from old regions at 1731476740990 (+4 ms)Region opened successfully at 1731476740995 (+5 ms) 2024-11-13T05:45:40,995 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c7b01a6d6207b5f0098648c3161dd6a6, disabling compactions & flushes 2024-11-13T05:45:40,995 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:40,995 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:40,996 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. after waiting 0 ms 2024-11-13T05:45:40,996 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:40,996 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:40,996 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c7b01a6d6207b5f0098648c3161dd6a6: Waiting for close lock at 1731476740995Disabling compacts and flushes for region at 1731476740995Disabling writes for close at 1731476740996 (+1 ms)Writing region close event to WAL at 1731476740996Closed at 1731476740996 2024-11-13T05:45:40,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741914_1092 (size=175) 2024-11-13T05:45:40,999 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-77232107/hregion-77232107.1731476740933 not finished, retry = 0 2024-11-13T05:45:40,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741914_1092 (size=175) 2024-11-13T05:45:40,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741914_1092 (size=175) 2024-11-13T05:45:41,102 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:41,103 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77232107:(num 1731476740933) 2024-11-13T05:45:41,103 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:41,106 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:41,121 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106, exclude list is [], retry=0 2024-11-13T05:45:41,124 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:41,124 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:41,124 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:41,147 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106 2024-11-13T05:45:41,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:41,148 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c7b01a6d6207b5f0098648c3161dd6a6, NAME => 'testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:41,148 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:41,148 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,148 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,149 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,150 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7b01a6d6207b5f0098648c3161dd6a6 columnFamilyName a 2024-11-13T05:45:41,150 DEBUG [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,151 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(327): Store=c7b01a6d6207b5f0098648c3161dd6a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,151 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,153 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,153 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,153 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,155 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,155 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c7b01a6d6207b5f0098648c3161dd6a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59367133, jitterRate=-0.11536078155040741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:41,156 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c7b01a6d6207b5f0098648c3161dd6a6: Writing region info on filesystem at 1731476741148Initializing all the Stores at 1731476741149 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741149Cleaning up temporary data from old regions at 1731476741153 (+4 ms)Region opened successfully at 1731476741156 (+3 ms) 2024-11-13T05:45:41,165 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c7b01a6d6207b5f0098648c3161dd6a6, disabling compactions & flushes 2024-11-13T05:45:41,165 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:41,165 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:41,165 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. after waiting 0 ms 2024-11-13T05:45:41,165 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:41,166 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:41,166 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. 2024-11-13T05:45:41,166 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c7b01a6d6207b5f0098648c3161dd6a6: Waiting for close lock at 1731476741165Disabling compacts and flushes for region at 1731476741165Disabling writes for close at 1731476741165Writing region close event to WAL at 1731476741166 (+1 ms)Closed at 1731476741166 2024-11-13T05:45:41,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741916_1094 (size=1795) 2024-11-13T05:45:41,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741916_1094 (size=1795) 2024-11-13T05:45:41,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741916_1094 (size=1795) 2024-11-13T05:45:41,189 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106, size=1.8 K (1795bytes) 2024-11-13T05:45:41,189 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106 2024-11-13T05:45:41,190 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106 after 1ms 2024-11-13T05:45:41,214 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:41,215 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106 took 26ms 2024-11-13T05:45:41,216 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106 so closing down 2024-11-13T05:45:41,217 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:41,218 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731476741106.temp 2024-11-13T05:45:41,242 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000003-wal.1731476741106.temp 2024-11-13T05:45:41,242 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:41,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741917_1095 (size=1795) 2024-11-13T05:45:41,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741917_1095 (size=1795) 2024-11-13T05:45:41,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741917_1095 (size=1795) 2024-11-13T05:45:41,248 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000003-wal.1731476741106.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:41,249 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000003-wal.1731476741106.temp to hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000012 2024-11-13T05:45:41,249 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-13T05:45:41,249 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106, journal: Splitting hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106, size=1.8 K (1795bytes) at 1731476741189Finishing writing output for hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106 so closing down at 1731476741216 (+27 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000003-wal.1731476741106.temp at 1731476741242 (+26 ms)3 split writer threads finished at 1731476741242Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000003-wal.1731476741106.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731476741248 (+6 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000003-wal.1731476741106.temp to hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000012 at 1731476741249 (+1 ms)Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731476741249 2024-11-13T05:45:41,251 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741106 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476741106 2024-11-13T05:45:41,252 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000012 2024-11-13T05:45:41,278 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:41,591 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:41,593 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:41,605 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741593, exclude list is [], retry=0 2024-11-13T05:45:41,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:41,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:41,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:41,634 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741593 2024-11-13T05:45:41,634 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:41,634 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c7b01a6d6207b5f0098648c3161dd6a6, NAME => 'testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:41,634 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:41,634 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,634 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,637 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,639 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7b01a6d6207b5f0098648c3161dd6a6 columnFamilyName a 2024-11-13T05:45:41,639 DEBUG [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,640 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(327): Store=c7b01a6d6207b5f0098648c3161dd6a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,640 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,640 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,642 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000012 2024-11-13T05:45:41,668 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:41,669 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000012 2024-11-13T05:45:41,669 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c7b01a6d6207b5f0098648c3161dd6a6 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T05:45:41,685 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/.tmp/a/754e00e7fa6940c4a77d3408d51e894f is 79, key is testDatalossWhenInputError/a:x0/1731476741156/Put/seqid=0 2024-11-13T05:45:41,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741919_1097 (size=5808) 2024-11-13T05:45:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741919_1097 (size=5808) 2024-11-13T05:45:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741919_1097 (size=5808) 2024-11-13T05:45:41,693 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/.tmp/a/754e00e7fa6940c4a77d3408d51e894f 2024-11-13T05:45:41,704 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/.tmp/a/754e00e7fa6940c4a77d3408d51e894f as hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/a/754e00e7fa6940c4a77d3408d51e894f 2024-11-13T05:45:41,713 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/a/754e00e7fa6940c4a77d3408d51e894f, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T05:45:41,714 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c7b01a6d6207b5f0098648c3161dd6a6 in 44ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T05:45:41,714 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/0000000000000000012 2024-11-13T05:45:41,715 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,715 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,718 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,720 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T05:45:41,721 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c7b01a6d6207b5f0098648c3161dd6a6; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58889062, jitterRate=-0.12248459458351135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:41,721 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c7b01a6d6207b5f0098648c3161dd6a6: Writing region info on filesystem at 1731476741634Initializing all the Stores at 1731476741637 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741637Obtaining lock to block concurrent updates at 1731476741669 (+32 ms)Preparing flush snapshotting stores in c7b01a6d6207b5f0098648c3161dd6a6 at 1731476741669Finished memstore snapshotting testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731476741669Flushing stores of testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6. at 1731476741669Flushing c7b01a6d6207b5f0098648c3161dd6a6/a: creating writer at 1731476741669Flushing c7b01a6d6207b5f0098648c3161dd6a6/a: appending metadata at 1731476741685 (+16 ms)Flushing c7b01a6d6207b5f0098648c3161dd6a6/a: closing flushed file at 1731476741685Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a284e47: reopening flushed file at 1731476741702 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c7b01a6d6207b5f0098648c3161dd6a6 in 44ms, sequenceid=12, compaction requested=false; wal=null at 1731476741714 (+12 ms)Cleaning up temporary data from old regions at 1731476741715 (+1 ms)Region opened successfully at 1731476741721 (+6 ms) 2024-11-13T05:45:41,725 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c7b01a6d6207b5f0098648c3161dd6a6, NAME => 'testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:41,725 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731476740929.c7b01a6d6207b5f0098648c3161dd6a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:41,725 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,725 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,727 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,728 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7b01a6d6207b5f0098648c3161dd6a6 columnFamilyName a 2024-11-13T05:45:41,728 DEBUG [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,733 DEBUG [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/a/754e00e7fa6940c4a77d3408d51e894f 2024-11-13T05:45:41,733 INFO [StoreOpener-c7b01a6d6207b5f0098648c3161dd6a6-1 {}] regionserver.HStore(327): Store=c7b01a6d6207b5f0098648c3161dd6a6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,734 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,734 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,736 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,736 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,736 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,738 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c7b01a6d6207b5f0098648c3161dd6a6 2024-11-13T05:45:41,740 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testDatalossWhenInputError/c7b01a6d6207b5f0098648c3161dd6a6/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T05:45:41,741 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c7b01a6d6207b5f0098648c3161dd6a6; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63913947, jitterRate=-0.04760797321796417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:45:41,741 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c7b01a6d6207b5f0098648c3161dd6a6: Writing region info on filesystem at 1731476741725Initializing all the Stores at 1731476741726 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741726Cleaning up temporary data from old regions at 1731476741736 (+10 ms)Region opened successfully at 1731476741741 (+5 ms) 2024-11-13T05:45:41,763 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:58888 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:32780 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:48284 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:48186 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-428696096-172.17.0.2-1731476706240:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:58826 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:32832 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 274), ProcessCount=11 (was 11), AvailableMemoryMB=7366 (was 7383) 2024-11-13T05:45:41,764 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-13T05:45:41,780 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=7359 2024-11-13T05:45:41,780 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-13T05:45:41,800 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:41,803 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:41,804 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:41,807 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08779094, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-08779094, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:41,820 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08779094/hregion-08779094.1731476741807, exclude list is [], retry=0 2024-11-13T05:45:41,824 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:41,824 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:41,825 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:41,853 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08779094/hregion-08779094.1731476741807 2024-11-13T05:45:41,854 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:41,854 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c467f37cd8e225a7ca8ef27974cea289, NAME => 'testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:41,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741921_1099 (size=63) 2024-11-13T05:45:41,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741921_1099 (size=63) 2024-11-13T05:45:41,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741921_1099 (size=63) 2024-11-13T05:45:41,865 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:41,866 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,867 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName a 2024-11-13T05:45:41,867 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,868 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,868 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,869 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName b 2024-11-13T05:45:41,869 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,870 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,870 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,871 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName c 2024-11-13T05:45:41,871 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,871 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,872 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,872 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,874 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,874 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,874 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:41,875 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,877 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:41,878 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c467f37cd8e225a7ca8ef27974cea289; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74343779, jitterRate=0.10780863463878632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:41,878 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c467f37cd8e225a7ca8ef27974cea289: Writing region info on filesystem at 1731476741865Initializing all the Stores at 1731476741866 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741866Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741866Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741866Cleaning up temporary data from old regions at 1731476741874 (+8 ms)Region opened successfully at 1731476741878 (+4 ms) 2024-11-13T05:45:41,878 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c467f37cd8e225a7ca8ef27974cea289, disabling compactions & flushes 2024-11-13T05:45:41,878 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:41,878 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:41,878 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. after waiting 0 ms 2024-11-13T05:45:41,878 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:41,879 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:41,879 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c467f37cd8e225a7ca8ef27974cea289: Waiting for close lock at 1731476741878Disabling compacts and flushes for region at 1731476741878Disabling writes for close at 1731476741878Writing region close event to WAL at 1731476741879 (+1 ms)Closed at 1731476741879 2024-11-13T05:45:41,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741920_1098 (size=175) 2024-11-13T05:45:41,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741920_1098 (size=175) 2024-11-13T05:45:41,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741920_1098 (size=175) 2024-11-13T05:45:41,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:41,883 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08779094:(num 1731476741807) 2024-11-13T05:45:41,883 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:41,884 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:41,897 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885, exclude list is [], retry=0 2024-11-13T05:45:41,899 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:41,900 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:41,900 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:41,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 2024-11-13T05:45:41,925 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:41,925 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c467f37cd8e225a7ca8ef27974cea289, NAME => 'testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:41,925 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:41,925 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,925 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,927 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,928 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName a 2024-11-13T05:45:41,928 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,928 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,928 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,929 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName b 2024-11-13T05:45:41,929 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,930 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,930 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,930 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName c 2024-11-13T05:45:41,931 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:41,931 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:41,931 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,932 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,933 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,934 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:41,936 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:41,936 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c467f37cd8e225a7ca8ef27974cea289; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74429892, jitterRate=0.10909181833267212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:41,937 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c467f37cd8e225a7ca8ef27974cea289: Writing region info on filesystem at 1731476741926Initializing all the Stores at 1731476741926Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741926Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741927 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476741927Cleaning up temporary data from old regions at 1731476741934 (+7 ms)Region opened successfully at 1731476741937 (+3 ms) 2024-11-13T05:45:41,940 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731476741940/Put/seqid=0 2024-11-13T05:45:41,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741923_1101 (size=4875) 2024-11-13T05:45:41,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741923_1101 (size=4875) 2024-11-13T05:45:41,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741923_1101 (size=4875) 2024-11-13T05:45:41,952 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731476741951/Put/seqid=0 2024-11-13T05:45:41,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741924_1102 (size=4875) 2024-11-13T05:45:41,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741924_1102 (size=4875) 2024-11-13T05:45:41,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741924_1102 (size=4875) 2024-11-13T05:45:41,961 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731476741960/Put/seqid=0 2024-11-13T05:45:41,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741925_1103 (size=4875) 2024-11-13T05:45:41,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741925_1103 (size=4875) 2024-11-13T05:45:41,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741925_1103 (size=4875) 2024-11-13T05:45:41,968 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c467f37cd8e225a7ca8ef27974cea289/a 2024-11-13T05:45:41,971 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T05:45:41,971 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T05:45:41,971 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c467f37cd8e225a7ca8ef27974cea289/a 2024-11-13T05:45:41,975 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T05:45:41,975 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T05:45:41,975 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c467f37cd8e225a7ca8ef27974cea289/a 2024-11-13T05:45:41,978 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T05:45:41,978 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T05:45:41,979 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c467f37cd8e225a7ca8ef27974cea289 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T05:45:41,993 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/.tmp/a/e72632ba885e44f3820efa066c444ab2 is 55, key is testCompactedBulkLoadedFiles/a:a/1731476741937/Put/seqid=0 2024-11-13T05:45:41,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741926_1104 (size=5107) 2024-11-13T05:45:41,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741926_1104 (size=5107) 2024-11-13T05:45:41,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741926_1104 (size=5107) 2024-11-13T05:45:41,999 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/.tmp/a/e72632ba885e44f3820efa066c444ab2 2024-11-13T05:45:42,004 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/.tmp/a/e72632ba885e44f3820efa066c444ab2 as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2 2024-11-13T05:45:42,010 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T05:45:42,011 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c467f37cd8e225a7ca8ef27974cea289 in 32ms, sequenceid=4, compaction requested=false 2024-11-13T05:45:42,012 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c467f37cd8e225a7ca8ef27974cea289: 2024-11-13T05:45:42,013 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_ 2024-11-13T05:45:42,014 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_ 2024-11-13T05:45:42,015 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_ 2024-11-13T05:45:42,015 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile0 into c467f37cd8e225a7ca8ef27974cea289/a as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_ - updating store file list. 2024-11-13T05:45:42,020 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3370f7889a64455190d860b37b9d655f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:42,020 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_ into c467f37cd8e225a7ca8ef27974cea289/a 2024-11-13T05:45:42,020 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile0 into c467f37cd8e225a7ca8ef27974cea289/a (new location: hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_) 2024-11-13T05:45:42,021 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile1 into c467f37cd8e225a7ca8ef27974cea289/a as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_ - updating store file list. 2024-11-13T05:45:42,025 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8d45015e02a240f991eb76ac0e0cb248_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:42,025 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_ into c467f37cd8e225a7ca8ef27974cea289/a 2024-11-13T05:45:42,025 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile1 into c467f37cd8e225a7ca8ef27974cea289/a (new location: hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_) 2024-11-13T05:45:42,026 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile2 into c467f37cd8e225a7ca8ef27974cea289/a as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_ - updating store file list. 2024-11-13T05:45:42,032 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:42,032 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_ into c467f37cd8e225a7ca8ef27974cea289/a 2024-11-13T05:45:42,032 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39659/hbase/testCompactedBulkLoadedFiles/hfile2 into c467f37cd8e225a7ca8ef27974cea289/a (new location: hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_) 2024-11-13T05:45:42,040 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T05:45:42,040 DEBUG [Time-limited test {}] regionserver.HStore(1541): c467f37cd8e225a7ca8ef27974cea289/a is initiating major compaction (all files) 2024-11-13T05:45:42,040 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c467f37cd8e225a7ca8ef27974cea289/a in testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:42,040 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_] into tmpdir=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/.tmp, totalSize=19.3 K 2024-11-13T05:45:42,041 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e72632ba885e44f3820efa066c444ab2, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731476741937 2024-11-13T05:45:42,041 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3370f7889a64455190d860b37b9d655f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T05:45:42,042 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8d45015e02a240f991eb76ac0e0cb248_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T05:45:42,042 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T05:45:42,057 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/.tmp/a/b87d1691ce3c4cd48e632782c37186d0 is 55, key is testCompactedBulkLoadedFiles/a:a/1731476741937/Put/seqid=0 2024-11-13T05:45:42,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741927_1105 (size=6154) 2024-11-13T05:45:42,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741927_1105 (size=6154) 2024-11-13T05:45:42,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741927_1105 (size=6154) 2024-11-13T05:45:42,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/.tmp/a/b87d1691ce3c4cd48e632782c37186d0 as hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/b87d1691ce3c4cd48e632782c37186d0 2024-11-13T05:45:42,084 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c467f37cd8e225a7ca8ef27974cea289/a of c467f37cd8e225a7ca8ef27974cea289 into b87d1691ce3c4cd48e632782c37186d0(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T05:45:42,084 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c467f37cd8e225a7ca8ef27974cea289: 2024-11-13T05:45:42,084 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T05:45:42,084 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T05:45:42,116 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885, size=0 (0bytes) 2024-11-13T05:45:42,116 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 might be still open, length is 0 2024-11-13T05:45:42,116 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 2024-11-13T05:45:42,116 WARN [IPC Server handler 3 on default port 39659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-13T05:45:42,117 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 after 0ms 2024-11-13T05:45:45,067 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:44798 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:46375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44798 dst: /127.0.0.1:46375 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46375 remote=/127.0.0.1:44798]. Total timeout mills is 60000, 57018 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:45,068 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:39202 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41687:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39202 dst: /127.0.0.1:41687 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:45,068 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:57464 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:42769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57464 dst: /127.0.0.1:42769 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:45,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741922_1106 (size=1794) 2024-11-13T05:45:45,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741922_1106 (size=1794) 2024-11-13T05:45:46,073 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:45:46,117 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 after 4001ms 2024-11-13T05:45:46,148 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:46,148 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 took 4033ms 2024-11-13T05:45:46,150 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885; continuing. 2024-11-13T05:45:46,150 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 so closing down 2024-11-13T05:45:46,150 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:46,151 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731476741885.temp 2024-11-13T05:45:46,177 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000003-wal.1731476741885.temp 2024-11-13T05:45:46,177 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:46,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741928_1107 (size=800) 2024-11-13T05:45:46,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741928_1107 (size=800) 2024-11-13T05:45:46,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741928_1107 (size=800) 2024-11-13T05:45:46,183 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000003-wal.1731476741885.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:46,184 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000003-wal.1731476741885.temp to hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000008 2024-11-13T05:45:46,185 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T05:45:46,185 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885, journal: Splitting hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885, size=0 (0bytes) at 1731476742116Finishing writing output for hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 so closing down at 1731476746150 (+4034 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000003-wal.1731476741885.temp at 1731476746177 (+27 ms)3 split writer threads finished at 1731476746177Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000003-wal.1731476741885.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731476746183 (+6 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000003-wal.1731476741885.temp to hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000008 at 1731476746184 (+1 ms)Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885, size=0, length=0, corrupted=false, cancelled=false at 1731476746185 (+1 ms) 2024-11-13T05:45:46,186 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476741885 2024-11-13T05:45:46,187 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000008 2024-11-13T05:45:46,187 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:46,189 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:46,202 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476746189, exclude list is [], retry=0 2024-11-13T05:45:46,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:46,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:46,205 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:46,227 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476746189 2024-11-13T05:45:46,228 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:46,228 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c467f37cd8e225a7ca8ef27974cea289, NAME => 'testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:46,228 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:46,228 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,228 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,229 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,230 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName a 2024-11-13T05:45:46,230 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,237 DEBUG [StoreFileOpener-c467f37cd8e225a7ca8ef27974cea289-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:46,237 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_ 2024-11-13T05:45:46,241 DEBUG [StoreFileOpener-c467f37cd8e225a7ca8ef27974cea289-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3370f7889a64455190d860b37b9d655f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:46,241 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_ 2024-11-13T05:45:46,245 DEBUG [StoreFileOpener-c467f37cd8e225a7ca8ef27974cea289-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8d45015e02a240f991eb76ac0e0cb248_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T05:45:46,245 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_ 2024-11-13T05:45:46,249 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/b87d1691ce3c4cd48e632782c37186d0 2024-11-13T05:45:46,252 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2 2024-11-13T05:45:46,252 WARN [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-11-13T05:45:46,252 WARN [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-11-13T05:45:46,253 WARN [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-11-13T05:45:46,253 WARN [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@ee55e31 2024-11-13T05:45:46,253 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2] to archive 2024-11-13T05:45:46,253 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T05:45:46,255 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_ to hdfs://localhost:39659/hbase/archive/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_ 2024-11-13T05:45:46,256 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_ to hdfs://localhost:39659/hbase/archive/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/3370f7889a64455190d860b37b9d655f_SeqId_4_ 2024-11-13T05:45:46,257 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_ to hdfs://localhost:39659/hbase/archive/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/8d45015e02a240f991eb76ac0e0cb248_SeqId_4_ 2024-11-13T05:45:46,258 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2 to hdfs://localhost:39659/hbase/archive/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/a/e72632ba885e44f3820efa066c444ab2 2024-11-13T05:45:46,258 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,258 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,259 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName b 2024-11-13T05:45:46,259 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,260 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,260 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,260 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c467f37cd8e225a7ca8ef27974cea289 columnFamilyName c 2024-11-13T05:45:46,260 DEBUG [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,261 INFO [StoreOpener-c467f37cd8e225a7ca8ef27974cea289-1 {}] regionserver.HStore(327): Store=c467f37cd8e225a7ca8ef27974cea289/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,261 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,262 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,264 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000008 2024-11-13T05:45:46,297 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c467f37cd8e225a7ca8ef27974cea289 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c467f37cd8e225a7ca8ef27974cea289" family_name: "a" compaction_input: "e72632ba885e44f3820efa066c444ab2" compaction_input: "3370f7889a64455190d860b37b9d655f_SeqId_4_" compaction_input: "8d45015e02a240f991eb76ac0e0cb248_SeqId_4_" compaction_input: "29ae96919ff741c6acb2c2bc23b3b24b_SeqId_4_" compaction_output: "b87d1691ce3c4cd48e632782c37186d0" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T05:45:46,299 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T05:45:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000008 2024-11-13T05:45:46,300 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/0000000000000000008 2024-11-13T05:45:46,301 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,301 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,302 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:46,303 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c467f37cd8e225a7ca8ef27974cea289 2024-11-13T05:45:46,306 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testCompactedBulkLoadedFiles/c467f37cd8e225a7ca8ef27974cea289/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:45:46,307 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c467f37cd8e225a7ca8ef27974cea289; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62545449, jitterRate=-0.0680001825094223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:46,307 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c467f37cd8e225a7ca8ef27974cea289: Writing region info on filesystem at 1731476746228Initializing all the Stores at 1731476746229 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746229Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746229Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746229Cleaning up temporary data from old regions at 1731476746301 (+72 ms)Region opened successfully at 1731476746307 (+6 ms) 2024-11-13T05:45:46,310 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c467f37cd8e225a7ca8ef27974cea289, disabling compactions & flushes 2024-11-13T05:45:46,310 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:46,310 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:46,310 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. after waiting 0 ms 2024-11-13T05:45:46,310 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:46,311 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731476741801.c467f37cd8e225a7ca8ef27974cea289. 2024-11-13T05:45:46,311 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c467f37cd8e225a7ca8ef27974cea289: Waiting for close lock at 1731476746310Disabling compacts and flushes for region at 1731476746310Disabling writes for close at 1731476746310Writing region close event to WAL at 1731476746311 (+1 ms)Closed at 1731476746311 2024-11-13T05:45:46,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741929_1108 (size=175) 2024-11-13T05:45:46,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741929_1108 (size=175) 2024-11-13T05:45:46,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741929_1108 (size=175) 2024-11-13T05:45:46,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:46,317 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731476746189) 2024-11-13T05:45:46,333 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1978960232_22 at /127.0.0.1:44860 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1978960232_22 at /127.0.0.1:39318 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:39659 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39659 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1978960232_22 at /127.0.0.1:57508 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7364 (was 7359) - AvailableMemoryMB LEAK? - 2024-11-13T05:45:46,333 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-13T05:45:46,348 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=7363 2024-11-13T05:45:46,348 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-13T05:45:46,363 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:46,365 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:45:46,365 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:45:46,368 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-88724748, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/hregion-88724748, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:46,380 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-88724748/hregion-88724748.1731476746368, exclude list is [], retry=0 2024-11-13T05:45:46,383 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:46,383 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:46,383 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:46,408 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-88724748/hregion-88724748.1731476746368 2024-11-13T05:45:46,409 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673)] 2024-11-13T05:45:46,409 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e24c12265fe67db8404d781a76bd1765, NAME => 'testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39659/hbase 2024-11-13T05:45:46,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741931_1110 (size=67) 2024-11-13T05:45:46,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741931_1110 (size=67) 2024-11-13T05:45:46,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741931_1110 (size=67) 2024-11-13T05:45:46,419 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:46,420 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,422 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName a 2024-11-13T05:45:46,422 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,422 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,422 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,423 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName b 2024-11-13T05:45:46,424 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,424 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,424 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,425 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName c 2024-11-13T05:45:46,425 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,426 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,426 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,427 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,427 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,428 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,429 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,429 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:46,430 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,432 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:45:46,433 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e24c12265fe67db8404d781a76bd1765; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62373821, jitterRate=-0.07055763900279999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:46,433 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e24c12265fe67db8404d781a76bd1765: Writing region info on filesystem at 1731476746419Initializing all the Stores at 1731476746420 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746420Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746420Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746420Cleaning up temporary data from old regions at 1731476746429 (+9 ms)Region opened successfully at 1731476746433 (+4 ms) 2024-11-13T05:45:46,433 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e24c12265fe67db8404d781a76bd1765, disabling compactions & flushes 2024-11-13T05:45:46,433 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,433 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,433 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. after waiting 0 ms 2024-11-13T05:45:46,433 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,433 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,433 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e24c12265fe67db8404d781a76bd1765: Waiting for close lock at 1731476746433Disabling compacts and flushes for region at 1731476746433Disabling writes for close at 1731476746433Writing region close event to WAL at 1731476746433Closed at 1731476746433 2024-11-13T05:45:46,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741930_1109 (size=175) 2024-11-13T05:45:46,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741930_1109 (size=175) 2024-11-13T05:45:46,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741930_1109 (size=175) 2024-11-13T05:45:46,438 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:46,438 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-88724748:(num 1731476746368) 2024-11-13T05:45:46,438 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:46,440 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:46,453 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440, exclude list is [], retry=0 2024-11-13T05:45:46,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:46,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:46,456 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:46,482 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440 2024-11-13T05:45:46,482 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:46,482 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e24c12265fe67db8404d781a76bd1765, NAME => 'testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:46,483 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:46,483 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,483 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,484 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,485 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName a 2024-11-13T05:45:46,485 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,486 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,486 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,487 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName b 2024-11-13T05:45:46,487 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,487 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,488 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,488 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName c 2024-11-13T05:45:46,488 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,489 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,489 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,490 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,491 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,492 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,492 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:46,493 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,493 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e24c12265fe67db8404d781a76bd1765; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63004515, jitterRate=-0.061159566044807434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:46,494 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e24c12265fe67db8404d781a76bd1765: Writing region info on filesystem at 1731476746483Initializing all the Stores at 1731476746484 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746484Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746484Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746484Cleaning up temporary data from old regions at 1731476746492 (+8 ms)Region opened successfully at 1731476746493 (+1 ms) 2024-11-13T05:45:46,501 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e24c12265fe67db8404d781a76bd1765 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T05:45:46,516 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/a/15670b9442094717bf22f3c199d3a463 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731476746494/Put/seqid=0 2024-11-13T05:45:46,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741933_1112 (size=5958) 2024-11-13T05:45:46,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741933_1112 (size=5958) 2024-11-13T05:45:46,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741933_1112 (size=5958) 2024-11-13T05:45:46,525 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/a/15670b9442094717bf22f3c199d3a463 2024-11-13T05:45:46,531 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/a/15670b9442094717bf22f3c199d3a463 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/a/15670b9442094717bf22f3c199d3a463 2024-11-13T05:45:46,536 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/a/15670b9442094717bf22f3c199d3a463, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T05:45:46,537 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e24c12265fe67db8404d781a76bd1765 in 36ms, sequenceid=13, compaction requested=false 2024-11-13T05:45:46,537 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e24c12265fe67db8404d781a76bd1765: 2024-11-13T05:45:46,552 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e24c12265fe67db8404d781a76bd1765, disabling compactions & flushes 2024-11-13T05:45:46,553 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,553 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,553 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. after waiting 0 ms 2024-11-13T05:45:46,553 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,553 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,553 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:46,553 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e24c12265fe67db8404d781a76bd1765: Waiting for close lock at 1731476746552Disabling compacts and flushes for region at 1731476746552Disabling writes for close at 1731476746553 (+1 ms)Writing region close event to WAL at 1731476746553Closed at 1731476746553 2024-11-13T05:45:46,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741932_1111 (size=6243) 2024-11-13T05:45:46,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741932_1111 (size=6243) 2024-11-13T05:45:46,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741932_1111 (size=6243) 2024-11-13T05:45:46,570 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440, size=6.1 K (6243bytes) 2024-11-13T05:45:46,570 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440 2024-11-13T05:45:46,570 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440 after 0ms 2024-11-13T05:45:46,594 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:46,594 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440 took 24ms 2024-11-13T05:45:46,596 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440 so closing down 2024-11-13T05:45:46,596 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:46,597 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731476746440.temp 2024-11-13T05:45:46,622 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000003-wal.1731476746440.temp 2024-11-13T05:45:46,622 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:46,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741934_1113 (size=5575) 2024-11-13T05:45:46,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741934_1113 (size=5575) 2024-11-13T05:45:46,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741934_1113 (size=5575) 2024-11-13T05:45:46,629 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000003-wal.1731476746440.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:46,630 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000003-wal.1731476746440.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000035 2024-11-13T05:45:46,630 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-13T05:45:46,630 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440, journal: Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440, size=6.1 K (6243bytes) at 1731476746570Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440 so closing down at 1731476746596 (+26 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000003-wal.1731476746440.temp at 1731476746622 (+26 ms)3 split writer threads finished at 1731476746622Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000003-wal.1731476746440.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731476746629 (+7 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000003-wal.1731476746440.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000035 at 1731476746630 (+1 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731476746630 2024-11-13T05:45:46,632 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746440 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476746440 2024-11-13T05:45:46,632 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000035 2024-11-13T05:45:46,632 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:46,634 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:46,647 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634, exclude list is [], retry=0 2024-11-13T05:45:46,649 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:46,649 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:46,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:46,676 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 2024-11-13T05:45:46,676 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43503:43503),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41547:41547)] 2024-11-13T05:45:46,676 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e24c12265fe67db8404d781a76bd1765, NAME => 'testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:45:46,676 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:46,676 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,676 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,678 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,679 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName a 2024-11-13T05:45:46,679 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,685 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/a/15670b9442094717bf22f3c199d3a463 2024-11-13T05:45:46,686 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,686 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,687 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName b 2024-11-13T05:45:46,687 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,687 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,687 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,688 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName c 2024-11-13T05:45:46,688 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:46,689 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:46,689 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,690 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,692 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000035 2024-11-13T05:45:46,720 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:46,721 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000035 2024-11-13T05:45:46,721 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e24c12265fe67db8404d781a76bd1765 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T05:45:46,735 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/b/4036b5d348b845f5b8e929bf02ad9cbe is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731476746537/Put/seqid=0 2024-11-13T05:45:46,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741936_1115 (size=5958) 2024-11-13T05:45:46,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741936_1115 (size=5958) 2024-11-13T05:45:46,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741936_1115 (size=5958) 2024-11-13T05:45:46,742 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/b/4036b5d348b845f5b8e929bf02ad9cbe 2024-11-13T05:45:46,761 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/c/9b31b69fa8854bf68d77474c351092c7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731476746544/Put/seqid=0 2024-11-13T05:45:46,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741937_1116 (size=5958) 2024-11-13T05:45:46,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741937_1116 (size=5958) 2024-11-13T05:45:46,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741937_1116 (size=5958) 2024-11-13T05:45:46,771 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/c/9b31b69fa8854bf68d77474c351092c7 2024-11-13T05:45:46,777 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/b/4036b5d348b845f5b8e929bf02ad9cbe as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/b/4036b5d348b845f5b8e929bf02ad9cbe 2024-11-13T05:45:46,783 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/b/4036b5d348b845f5b8e929bf02ad9cbe, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T05:45:46,784 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/c/9b31b69fa8854bf68d77474c351092c7 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/c/9b31b69fa8854bf68d77474c351092c7 2024-11-13T05:45:46,789 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/c/9b31b69fa8854bf68d77474c351092c7, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T05:45:46,789 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e24c12265fe67db8404d781a76bd1765 in 68ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T05:45:46,790 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000035 2024-11-13T05:45:46,791 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,791 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,791 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:46,793 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:46,795 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T05:45:46,796 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e24c12265fe67db8404d781a76bd1765; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74869218, jitterRate=0.11563828587532043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:46,796 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e24c12265fe67db8404d781a76bd1765: Writing region info on filesystem at 1731476746677Initializing all the Stores at 1731476746677Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746677Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746678 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476746678Obtaining lock to block concurrent updates at 1731476746721 (+43 ms)Preparing flush snapshotting stores in e24c12265fe67db8404d781a76bd1765 at 1731476746721Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731476746721Flushing stores of testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. at 1731476746721Flushing e24c12265fe67db8404d781a76bd1765/b: creating writer at 1731476746721Flushing e24c12265fe67db8404d781a76bd1765/b: appending metadata at 1731476746735 (+14 ms)Flushing e24c12265fe67db8404d781a76bd1765/b: closing flushed file at 1731476746735Flushing e24c12265fe67db8404d781a76bd1765/c: creating writer at 1731476746746 (+11 ms)Flushing e24c12265fe67db8404d781a76bd1765/c: appending metadata at 1731476746760 (+14 ms)Flushing e24c12265fe67db8404d781a76bd1765/c: closing flushed file at 1731476746761 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41b69eed: reopening flushed file at 1731476746776 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135ddfac: reopening flushed file at 1731476746783 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e24c12265fe67db8404d781a76bd1765 in 68ms, sequenceid=35, compaction requested=false; wal=null at 1731476746789 (+6 ms)Cleaning up temporary data from old regions at 1731476746791 (+2 ms)Region opened successfully at 1731476746796 (+5 ms) 2024-11-13T05:45:46,858 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634, size=0 (0bytes) 2024-11-13T05:45:46,858 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 might be still open, length is 0 2024-11-13T05:45:46,858 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 2024-11-13T05:45:46,859 WARN [IPC Server handler 1 on default port 39659 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-13T05:45:46,859 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 after 1ms 2024-11-13T05:45:48,068 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:57594 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:42769:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57594 dst: /127.0.0.1:42769 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42769 remote=/127.0.0.1:57594]. Total timeout mills is 60000, 58754 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:48,069 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:39362 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41687:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39362 dst: /127.0.0.1:41687 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:48,069 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-146518280_22 at /127.0.0.1:44902 [Receiving block BP-428696096-172.17.0.2-1731476706240:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:46375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44902 dst: /127.0.0.1:46375 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:48,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741935_1117 (size=5567) 2024-11-13T05:45:48,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741935_1117 (size=5567) 2024-11-13T05:45:48,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741935_1117 (size=5567) 2024-11-13T05:45:50,286 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T05:45:50,286 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:50,287 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T05:45:50,287 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T05:45:50,860 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 after 4002ms 2024-11-13T05:45:50,892 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:50,892 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 took 4034ms 2024-11-13T05:45:50,894 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634; continuing. 2024-11-13T05:45:50,894 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 so closing down 2024-11-13T05:45:50,895 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:45:50,896 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731476746634.temp 2024-11-13T05:45:50,919 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000037-wal.1731476746634.temp 2024-11-13T05:45:50,919 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:45:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741938_1118 (size=5575) 2024-11-13T05:45:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741938_1118 (size=5575) 2024-11-13T05:45:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741938_1118 (size=5575) 2024-11-13T05:45:50,926 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000037-wal.1731476746634.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T05:45:50,927 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000037-wal.1731476746634.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000066 2024-11-13T05:45:50,927 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T05:45:50,927 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634, journal: Splitting hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634, size=0 (0bytes) at 1731476746858Finishing writing output for hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 so closing down at 1731476750894 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000037-wal.1731476746634.temp at 1731476750919 (+25 ms)3 split writer threads finished at 1731476750919Closed recovered edits writer path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000037-wal.1731476746634.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731476750926 (+7 ms)Rename recovered edits hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000037-wal.1731476746634.temp to hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000066 at 1731476750927 (+1 ms)Processed 30 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634, size=0, length=0, corrupted=false, cancelled=false at 1731476750927 2024-11-13T05:45:50,929 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 to hdfs://localhost:39659/hbase/oldWALs/wal.1731476746634 2024-11-13T05:45:50,930 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000066 2024-11-13T05:45:50,930 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T05:45:50,932 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39659/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362, archiveDir=hdfs://localhost:39659/hbase/oldWALs, maxLogs=32 2024-11-13T05:45:50,944 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476750932, exclude list is [], retry=0 2024-11-13T05:45:50,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41687,DS-f9c6cef8-ddb9-4773-a618-d0e954165336,DISK] 2024-11-13T05:45:50,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46375,DS-a5bab832-1486-42a2-b8c2-317310a1a45d,DISK] 2024-11-13T05:45:50,947 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42769,DS-01408029-007f-41cd-ad85-d56957288564,DISK] 2024-11-13T05:45:50,972 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476750932 2024-11-13T05:45:50,972 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41547:41547),(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:43503:43503)] 2024-11-13T05:45:50,972 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:45:50,974 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:50,974 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName a 2024-11-13T05:45:50,974 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:50,979 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/a/15670b9442094717bf22f3c199d3a463 2024-11-13T05:45:50,979 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:50,979 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:50,980 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName b 2024-11-13T05:45:50,980 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:50,986 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/b/4036b5d348b845f5b8e929bf02ad9cbe 2024-11-13T05:45:50,986 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:50,986 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:50,987 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e24c12265fe67db8404d781a76bd1765 columnFamilyName c 2024-11-13T05:45:50,987 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:45:50,992 DEBUG [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/c/9b31b69fa8854bf68d77474c351092c7 2024-11-13T05:45:50,992 INFO [StoreOpener-e24c12265fe67db8404d781a76bd1765-1 {}] regionserver.HStore(327): Store=e24c12265fe67db8404d781a76bd1765/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:45:50,992 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:50,993 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:50,995 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:50,995 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000066 2024-11-13T05:45:51,021 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T05:45:51,024 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000066 2024-11-13T05:45:51,024 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e24c12265fe67db8404d781a76bd1765 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T05:45:51,038 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/a/4382eb99bebc495097e4842d49256039 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731476746803/Put/seqid=0 2024-11-13T05:45:51,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741940_1120 (size=5958) 2024-11-13T05:45:51,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741940_1120 (size=5958) 2024-11-13T05:45:51,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741940_1120 (size=5958) 2024-11-13T05:45:51,045 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/a/4382eb99bebc495097e4842d49256039 2024-11-13T05:45:51,063 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/b/4eb0c57e75ed43fab2dc0260d0589e96 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731476746809/Put/seqid=0 2024-11-13T05:45:51,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741941_1121 (size=5958) 2024-11-13T05:45:51,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741941_1121 (size=5958) 2024-11-13T05:45:51,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741941_1121 (size=5958) 2024-11-13T05:45:51,070 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/b/4eb0c57e75ed43fab2dc0260d0589e96 2024-11-13T05:45:51,089 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/c/11536a02035b40059fd6fadb4a33e1f1 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731476746815/Put/seqid=0 2024-11-13T05:45:51,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741942_1122 (size=5958) 2024-11-13T05:45:51,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741942_1122 (size=5958) 2024-11-13T05:45:51,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741942_1122 (size=5958) 2024-11-13T05:45:51,096 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/c/11536a02035b40059fd6fadb4a33e1f1 2024-11-13T05:45:51,101 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/a/4382eb99bebc495097e4842d49256039 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/a/4382eb99bebc495097e4842d49256039 2024-11-13T05:45:51,105 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/a/4382eb99bebc495097e4842d49256039, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T05:45:51,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/b/4eb0c57e75ed43fab2dc0260d0589e96 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/b/4eb0c57e75ed43fab2dc0260d0589e96 2024-11-13T05:45:51,112 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/b/4eb0c57e75ed43fab2dc0260d0589e96, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T05:45:51,113 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/.tmp/c/11536a02035b40059fd6fadb4a33e1f1 as hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/c/11536a02035b40059fd6fadb4a33e1f1 2024-11-13T05:45:51,117 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/c/11536a02035b40059fd6fadb4a33e1f1, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T05:45:51,117 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e24c12265fe67db8404d781a76bd1765 in 93ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T05:45:51,118 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/0000000000000000066 2024-11-13T05:45:51,119 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:51,119 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:51,119 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:45:51,120 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e24c12265fe67db8404d781a76bd1765 2024-11-13T05:45:51,122 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/hbase/data/default/testReplayEditsWrittenViaHRegion/e24c12265fe67db8404d781a76bd1765/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T05:45:51,123 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e24c12265fe67db8404d781a76bd1765; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73725560, jitterRate=0.09859645366668701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:45:51,123 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e24c12265fe67db8404d781a76bd1765: Writing region info on filesystem at 1731476750972Initializing all the Stores at 1731476750973 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476750973Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476750973Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731476750973Obtaining lock to block concurrent updates at 1731476751024 (+51 ms)Preparing flush snapshotting stores in e24c12265fe67db8404d781a76bd1765 at 1731476751024Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731476751024Flushing stores of testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. at 1731476751024Flushing e24c12265fe67db8404d781a76bd1765/a: creating writer at 1731476751024Flushing e24c12265fe67db8404d781a76bd1765/a: appending metadata at 1731476751037 (+13 ms)Flushing e24c12265fe67db8404d781a76bd1765/a: closing flushed file at 1731476751037Flushing e24c12265fe67db8404d781a76bd1765/b: creating writer at 1731476751049 (+12 ms)Flushing e24c12265fe67db8404d781a76bd1765/b: appending metadata at 1731476751063 (+14 ms)Flushing e24c12265fe67db8404d781a76bd1765/b: closing flushed file at 1731476751063Flushing e24c12265fe67db8404d781a76bd1765/c: creating writer at 1731476751075 (+12 ms)Flushing e24c12265fe67db8404d781a76bd1765/c: appending metadata at 1731476751089 (+14 ms)Flushing e24c12265fe67db8404d781a76bd1765/c: closing flushed file at 1731476751089Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d93b3da: reopening flushed file at 1731476751100 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a6c5ef3: reopening flushed file at 1731476751106 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bdb1bb6: reopening flushed file at 1731476751112 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e24c12265fe67db8404d781a76bd1765 in 93ms, sequenceid=66, compaction requested=false; wal=null at 1731476751117 (+5 ms)Cleaning up temporary data from old regions at 1731476751119 (+2 ms)Region opened successfully at 1731476751123 (+4 ms) 2024-11-13T05:45:51,136 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e24c12265fe67db8404d781a76bd1765, disabling compactions & flushes 2024-11-13T05:45:51,136 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:51,136 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:51,136 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. after waiting 0 ms 2024-11-13T05:45:51,136 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:51,137 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731476746363.e24c12265fe67db8404d781a76bd1765. 2024-11-13T05:45:51,137 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e24c12265fe67db8404d781a76bd1765: Waiting for close lock at 1731476751136Disabling compacts and flushes for region at 1731476751136Disabling writes for close at 1731476751136Writing region close event to WAL at 1731476751137 (+1 ms)Closed at 1731476751137 2024-11-13T05:45:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741939_1119 (size=175) 2024-11-13T05:45:51,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741939_1119 (size=175) 2024-11-13T05:45:51,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741939_1119 (size=175) 2024-11-13T05:45:51,141 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T05:45:51,141 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731476750932) 2024-11-13T05:45:51,153 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=442 (was 438) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_16647536_22 at /127.0.0.1:57658 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_16647536_22 at /127.0.0.1:39414 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:39659 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_16647536_22 at /127.0.0.1:44936 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39659 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=7353 (was 7363) 2024-11-13T05:45:51,153 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-13T05:45:51,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:45:51,154 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:45:51,154 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:45:51,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:51,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:51,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:45:51,154 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:45:51,154 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=481480576, stopped=false 2024-11-13T05:45:51,155 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bc386b7bf2fe,38153,1731476709944 2024-11-13T05:45:51,207 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:45:51,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:45:51,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:45:51,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:51,208 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:51,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:51,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:45:51,208 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:45:51,208 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:45:51,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:51,208 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,45573,1731476710694' ***** 2024-11-13T05:45:51,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:45:51,208 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:45:51,208 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,46279,1731476710827' ***** 2024-11-13T05:45:51,208 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:45:51,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:45:51,209 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:45:51,209 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:45:51,209 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:45:51,209 INFO [RS:0;bc386b7bf2fe:45573 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:45:51,209 INFO [RS:0;bc386b7bf2fe:45573 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:45:51,209 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:45:51,209 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(3091): Received CLOSE for 06e18f488ac3dcef5cfa0c667ec59dce 2024-11-13T05:45:51,209 INFO [RS:1;bc386b7bf2fe:46279 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:45:51,209 INFO [RS:1;bc386b7bf2fe:46279 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:45:51,209 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:45:51,209 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:51,209 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:51,209 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:45:51,209 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:45:51,209 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06e18f488ac3dcef5cfa0c667ec59dce, disabling compactions & flushes 2024-11-13T05:45:51,209 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:51,209 INFO [RS:1;bc386b7bf2fe:46279 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bc386b7bf2fe:46279. 2024-11-13T05:45:51,209 INFO [RS:0;bc386b7bf2fe:45573 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bc386b7bf2fe:45573. 2024-11-13T05:45:51,209 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:51,209 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. after waiting 0 ms 2024-11-13T05:45:51,209 DEBUG [RS:1;bc386b7bf2fe:46279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:45:51,209 DEBUG [RS:0;bc386b7bf2fe:45573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:45:51,209 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:51,210 DEBUG [RS:1;bc386b7bf2fe:46279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:51,210 DEBUG [RS:0;bc386b7bf2fe:45573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:51,210 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,46279,1731476710827; all regions closed. 2024-11-13T05:45:51,210 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:45:51,210 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:45:51,210 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:45:51,210 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:45:51,210 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:45:51,210 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1325): Online Regions={06e18f488ac3dcef5cfa0c667ec59dce=testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce., 1588230740=hbase:meta,,1.1588230740} 2024-11-13T05:45:51,210 DEBUG [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1351): Waiting on 06e18f488ac3dcef5cfa0c667ec59dce, 1588230740 2024-11-13T05:45:51,210 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:45:51,210 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:45:51,210 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:45:51,210 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:45:51,211 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:45:51,211 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-13T05:45:51,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741833_1009 (size=175) 2024-11-13T05:45:51,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741833_1009 (size=175) 2024-11-13T05:45:51,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741833_1009 (size=175) 2024-11-13T05:45:51,214 DEBUG [RS:1;bc386b7bf2fe:46279 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs 2024-11-13T05:45:51,214 INFO [RS:1;bc386b7bf2fe:46279 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C46279%2C1731476710827:(num 1731476712487) 2024-11-13T05:45:51,214 DEBUG [RS:1;bc386b7bf2fe:46279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:51,214 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:45:51,214 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:45:51,215 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06e18f488ac3dcef5cfa0c667ec59dce/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T05:45:51,215 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:45:51,215 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:45:51,215 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:45:51,215 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:45:51,215 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:45:51,215 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:45:51,215 INFO [RS:1;bc386b7bf2fe:46279 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46279 2024-11-13T05:45:51,216 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/info/0c7729078a0f4e4fbb6d30d6485f22d3 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce./info:regioninfo/1731476730123/Put/seqid=0 2024-11-13T05:45:51,218 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:51,218 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06e18f488ac3dcef5cfa0c667ec59dce: Waiting for close lock at 1731476751209Running coprocessor pre-close hooks at 1731476751209Disabling compacts and flushes for region at 1731476751209Disabling writes for close at 1731476751209Writing region close event to WAL at 1731476751210 (+1 ms)Running coprocessor post-close hooks at 1731476751218 (+8 ms)Closed at 1731476751218 2024-11-13T05:45:51,219 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731476726026.06e18f488ac3dcef5cfa0c667ec59dce. 2024-11-13T05:45:51,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741943_1123 (size=6778) 2024-11-13T05:45:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741943_1123 (size=6778) 2024-11-13T05:45:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741943_1123 (size=6778) 2024-11-13T05:45:51,223 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/info/0c7729078a0f4e4fbb6d30d6485f22d3 2024-11-13T05:45:51,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,46279,1731476710827 2024-11-13T05:45:51,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:45:51,224 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:45:51,229 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/.tmp/info/0c7729078a0f4e4fbb6d30d6485f22d3 as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/info/0c7729078a0f4e4fbb6d30d6485f22d3 2024-11-13T05:45:51,232 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,46279,1731476710827] 2024-11-13T05:45:51,235 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/info/0c7729078a0f4e4fbb6d30d6485f22d3, entries=8, sequenceid=23, filesize=6.6 K 2024-11-13T05:45:51,236 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-11-13T05:45:51,236 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:45:51,241 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-13T05:45:51,241 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,46279,1731476710827 already deleted, retry=false 2024-11-13T05:45:51,241 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,46279,1731476710827 expired; onlineServers=1 2024-11-13T05:45:51,241 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:45:51,241 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:45:51,241 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731476751210Running coprocessor pre-close hooks at 1731476751210Disabling compacts and flushes for region at 1731476751210Disabling writes for close at 1731476751210Obtaining lock to block concurrent updates at 1731476751211 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731476751211Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731476751211Flushing stores of hbase:meta,,1.1588230740 at 1731476751212 (+1 ms)Flushing 1588230740/info: creating writer at 1731476751212Flushing 1588230740/info: appending metadata at 1731476751215 (+3 ms)Flushing 1588230740/info: closing flushed file at 1731476751215Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e7c8e8: reopening flushed file at 1731476751228 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1731476751236 (+8 ms)Writing region close event to WAL at 1731476751237 (+1 ms)Running coprocessor post-close hooks at 1731476751241 (+4 ms)Closed at 1731476751241 2024-11-13T05:45:51,241 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:45:51,273 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:45:51,273 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:45:51,329 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:45:51,330 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:45:51,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:51,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46279-0x101340371ed0002, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:51,333 INFO [RS:1;bc386b7bf2fe:46279 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:45:51,333 INFO [RS:1;bc386b7bf2fe:46279 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,46279,1731476710827; zookeeper connection closed. 2024-11-13T05:45:51,333 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53a914d7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53a914d7 2024-11-13T05:45:51,410 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,45573,1731476710694; all regions closed. 2024-11-13T05:45:51,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741893_1071 (size=2625) 2024-11-13T05:45:51,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741893_1071 (size=2625) 2024-11-13T05:45:51,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741893_1071 (size=2625) 2024-11-13T05:45:51,419 DEBUG [RS:0;bc386b7bf2fe:45573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs 2024-11-13T05:45:51,419 INFO [RS:0;bc386b7bf2fe:45573 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45573%2C1731476710694.meta:.meta(num 1731476729192) 2024-11-13T05:45:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741834_1010 (size=3621) 2024-11-13T05:45:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741834_1010 (size=3621) 2024-11-13T05:45:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741834_1010 (size=3621) 2024-11-13T05:45:51,424 DEBUG [RS:0;bc386b7bf2fe:45573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/oldWALs 2024-11-13T05:45:51,424 INFO [RS:0;bc386b7bf2fe:45573 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45573%2C1731476710694:(num 1731476712488) 2024-11-13T05:45:51,424 DEBUG [RS:0;bc386b7bf2fe:45573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:45:51,424 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:45:51,424 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:45:51,424 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T05:45:51,425 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:45:51,425 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:45:51,425 INFO [RS:0;bc386b7bf2fe:45573 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45573 2024-11-13T05:45:51,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:45:51,456 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,45573,1731476710694 2024-11-13T05:45:51,456 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:45:51,457 ERROR [pool-64-thread-1-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$369/0x00007f18b48e2b78@6ca45701 rejected from java.util.concurrent.ThreadPoolExecutor@29a6d4f1[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 16] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-13T05:45:51,466 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,45573,1731476710694] 2024-11-13T05:45:51,474 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,45573,1731476710694 already deleted, retry=false 2024-11-13T05:45:51,474 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,45573,1731476710694 expired; onlineServers=0 2024-11-13T05:45:51,475 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bc386b7bf2fe,38153,1731476709944' ***** 2024-11-13T05:45:51,475 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:45:51,475 INFO [M:0;bc386b7bf2fe:38153 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:45:51,475 INFO [M:0;bc386b7bf2fe:38153 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:45:51,475 DEBUG [M:0;bc386b7bf2fe:38153 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:45:51,476 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:45:51,476 DEBUG [M:0;bc386b7bf2fe:38153 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:45:51,476 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731476712068 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731476712068,5,FailOnTimeoutGroup] 2024-11-13T05:45:51,476 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731476712069 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731476712069,5,FailOnTimeoutGroup] 2024-11-13T05:45:51,476 INFO [M:0;bc386b7bf2fe:38153 {}] hbase.ChoreService(370): Chore service for: master/bc386b7bf2fe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:45:51,477 INFO [M:0;bc386b7bf2fe:38153 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:45:51,477 DEBUG [M:0;bc386b7bf2fe:38153 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:45:51,477 INFO [M:0;bc386b7bf2fe:38153 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:45:51,477 INFO [M:0;bc386b7bf2fe:38153 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:45:51,478 INFO [M:0;bc386b7bf2fe:38153 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:45:51,479 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:45:51,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:45:51,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:45:51,483 DEBUG [M:0;bc386b7bf2fe:38153 {}] zookeeper.ZKUtil(347): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:45:51,483 WARN [M:0;bc386b7bf2fe:38153 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:45:51,483 INFO [M:0;bc386b7bf2fe:38153 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/.lastflushedseqids 2024-11-13T05:45:51,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741944_1124 (size=119) 2024-11-13T05:45:51,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741944_1124 (size=119) 2024-11-13T05:45:51,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741944_1124 (size=119) 2024-11-13T05:45:51,497 INFO [M:0;bc386b7bf2fe:38153 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:45:51,497 INFO [M:0;bc386b7bf2fe:38153 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:45:51,497 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:45:51,497 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:51,497 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:51,497 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:45:51,497 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:51,497 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-11-13T05:45:51,513 DEBUG [M:0;bc386b7bf2fe:38153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/95e89f6bff5b4ce88431a4af2d2fc2a4 is 82, key is hbase:meta,,1/info:regioninfo/1731476729399/Put/seqid=0 2024-11-13T05:45:51,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741945_1125 (size=6063) 2024-11-13T05:45:51,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741945_1125 (size=6063) 2024-11-13T05:45:51,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741945_1125 (size=6063) 2024-11-13T05:45:51,521 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/95e89f6bff5b4ce88431a4af2d2fc2a4 2024-11-13T05:45:51,540 DEBUG [M:0;bc386b7bf2fe:38153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/453e084d518242adbc2b15ad73fc3b7f is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731476726455/Put/seqid=0 2024-11-13T05:45:51,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741946_1126 (size=7906) 2024-11-13T05:45:51,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741946_1126 (size=7906) 2024-11-13T05:45:51,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741946_1126 (size=7906) 2024-11-13T05:45:51,549 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/453e084d518242adbc2b15ad73fc3b7f 2024-11-13T05:45:51,553 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 453e084d518242adbc2b15ad73fc3b7f 2024-11-13T05:45:51,566 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:51,566 INFO [RS:0;bc386b7bf2fe:45573 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:45:51,566 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45573-0x101340371ed0001, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:51,566 INFO [RS:0;bc386b7bf2fe:45573 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,45573,1731476710694; zookeeper connection closed. 2024-11-13T05:45:51,566 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4737a9aa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4737a9aa 2024-11-13T05:45:51,567 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T05:45:51,573 DEBUG [M:0;bc386b7bf2fe:38153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6b119ab5cf944dd9a498b15f6e6a972 is 69, key is bc386b7bf2fe,45573,1731476710694/rs:state/1731476712194/Put/seqid=0 2024-11-13T05:45:51,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741947_1127 (size=5440) 2024-11-13T05:45:51,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741947_1127 (size=5440) 2024-11-13T05:45:51,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741947_1127 (size=5440) 2024-11-13T05:45:51,579 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6b119ab5cf944dd9a498b15f6e6a972 2024-11-13T05:45:51,584 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f6b119ab5cf944dd9a498b15f6e6a972 2024-11-13T05:45:51,585 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/95e89f6bff5b4ce88431a4af2d2fc2a4 as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/95e89f6bff5b4ce88431a4af2d2fc2a4 2024-11-13T05:45:51,590 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/95e89f6bff5b4ce88431a4af2d2fc2a4, entries=14, sequenceid=207, filesize=5.9 K 2024-11-13T05:45:51,591 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/453e084d518242adbc2b15ad73fc3b7f as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/453e084d518242adbc2b15ad73fc3b7f 2024-11-13T05:45:51,595 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 453e084d518242adbc2b15ad73fc3b7f 2024-11-13T05:45:51,595 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/453e084d518242adbc2b15ad73fc3b7f, entries=21, sequenceid=207, filesize=7.7 K 2024-11-13T05:45:51,596 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6b119ab5cf944dd9a498b15f6e6a972 as hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6b119ab5cf944dd9a498b15f6e6a972 2024-11-13T05:45:51,600 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f6b119ab5cf944dd9a498b15f6e6a972 2024-11-13T05:45:51,600 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39659/user/jenkins/test-data/7fdf4feb-fc4d-2993-c02c-bf942565f63d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6b119ab5cf944dd9a498b15f6e6a972, entries=3, sequenceid=207, filesize=5.3 K 2024-11-13T05:45:51,601 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=207, compaction requested=false 2024-11-13T05:45:51,602 INFO [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:45:51,602 DEBUG [M:0;bc386b7bf2fe:38153 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731476751497Disabling compacts and flushes for region at 1731476751497Disabling writes for close at 1731476751497Obtaining lock to block concurrent updates at 1731476751497Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731476751497Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1731476751497Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731476751498 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731476751498Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731476751513 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731476751513Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731476751526 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731476751540 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731476751540Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731476751553 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731476751572 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731476751572Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@678eeb7e: reopening flushed file at 1731476751584 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aab0448: reopening flushed file at 1731476751590 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14c0e4cf: reopening flushed file at 1731476751596 (+6 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=207, compaction requested=false at 1731476751601 (+5 ms)Writing region close event to WAL at 1731476751602 (+1 ms)Closed at 1731476751602 2024-11-13T05:45:51,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41687 is added to blk_1073741830_1006 (size=102150) 2024-11-13T05:45:51,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42769 is added to blk_1073741830_1006 (size=102150) 2024-11-13T05:45:51,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46375 is added to blk_1073741830_1006 (size=102150) 2024-11-13T05:45:51,605 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:45:51,605 INFO [M:0;bc386b7bf2fe:38153 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:45:51,605 INFO [M:0;bc386b7bf2fe:38153 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38153 2024-11-13T05:45:51,606 INFO [M:0;bc386b7bf2fe:38153 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:45:51,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:51,725 INFO [M:0;bc386b7bf2fe:38153 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:45:51,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38153-0x101340371ed0000, quorum=127.0.0.1:60627, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:45:51,736 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476740656 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476740656 (inode 16688) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731476731312/wal.1731476740656 (inode 16688) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T05:45:51,739 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476731118 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476731118 (inode 16665) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731476730643/wal.1731476731118 (inode 16665) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T05:45:51,739 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731476721501/wal.1731476721626 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:51,741 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741593 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741593 (inode 16714) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731476740928/wal.1731476741593 (inode 16714) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T05:45:51,742 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731476741800/wal.1731476741885 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:51,742 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731476746362/wal.1731476746634 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:51,743 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731476714668/wal.1731476714763 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:45:51,745 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal.1731476714482 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal.1731476714482 (inode 16485) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731476714105/wal.1731476714482 (inode 16485) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T05:45:51,747 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal.1731476713836 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal.1731476713836 (inode 16462) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731476713419/wal.1731476713836 (inode 16462) Holder DFSClient_NONMAPREDUCE_-146518280_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T05:45:51,751 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:45:51,753 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:45:51,753 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:45:51,754 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:45:51,754 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,STOPPED} 2024-11-13T05:45:51,756 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:45:51,756 WARN [BP-428696096-172.17.0.2-1731476706240 heartbeating to localhost/127.0.0.1:39659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:45:51,756 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:45:51,756 WARN [BP-428696096-172.17.0.2-1731476706240 heartbeating to localhost/127.0.0.1:39659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-428696096-172.17.0.2-1731476706240 (Datanode Uuid c7c96584-0d7b-4c3e-b65d-ead4dc9341ac) service to localhost/127.0.0.1:39659 2024-11-13T05:45:51,757 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data5/current/BP-428696096-172.17.0.2-1731476706240 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:45:51,758 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data6/current/BP-428696096-172.17.0.2-1731476706240 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:45:51,758 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:45:51,760 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:45:51,760 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:45:51,760 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:45:51,760 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:45:51,760 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,STOPPED} 2024-11-13T05:45:51,761 WARN [BP-428696096-172.17.0.2-1731476706240 heartbeating to localhost/127.0.0.1:39659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:45:51,761 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:45:51,761 WARN [BP-428696096-172.17.0.2-1731476706240 heartbeating to localhost/127.0.0.1:39659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-428696096-172.17.0.2-1731476706240 (Datanode Uuid b2f5bc00-560f-4425-b733-300d07a1a832) service to localhost/127.0.0.1:39659 2024-11-13T05:45:51,762 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:45:51,762 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data3/current/BP-428696096-172.17.0.2-1731476706240 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:45:51,762 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data4/current/BP-428696096-172.17.0.2-1731476706240 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:45:51,763 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:45:51,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:45:51,765 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:45:51,765 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:45:51,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:45:51,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,STOPPED} 2024-11-13T05:45:51,766 WARN [BP-428696096-172.17.0.2-1731476706240 heartbeating to localhost/127.0.0.1:39659 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:45:51,766 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:45:51,766 WARN [BP-428696096-172.17.0.2-1731476706240 heartbeating to localhost/127.0.0.1:39659 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-428696096-172.17.0.2-1731476706240 (Datanode Uuid f85652d7-75af-40d2-be7b-e1cff6d44d06) service to localhost/127.0.0.1:39659 2024-11-13T05:45:51,766 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:45:51,767 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data1/current/BP-428696096-172.17.0.2-1731476706240 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:45:51,767 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/cluster_e7bdb585-112f-10c5-40d0-99569ba4dfb4/data/data2/current/BP-428696096-172.17.0.2-1731476706240 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:45:51,767 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:45:51,774 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:45:51,774 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:45:51,774 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:45:51,775 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:45:51,775 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eec51dec-9919-f736-9d73-7c33f8fdf44a/hadoop.log.dir/,STOPPED} 2024-11-13T05:45:51,784 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:45:51,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down