2024-11-20 21:28:30,753 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 21:28:30,767 main DEBUG Took 0.012538 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 21:28:30,768 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 21:28:30,768 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 21:28:30,769 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 21:28:30,770 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,777 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 21:28:30,797 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,799 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,800 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,801 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,802 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,804 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,806 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,807 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,808 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,809 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,810 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:28:30,811 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,811 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 21:28:30,813 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:28:30,815 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 21:28:30,818 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 21:28:30,818 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 21:28:30,819 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 21:28:30,820 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 21:28:30,831 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 21:28:30,834 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 21:28:30,836 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 21:28:30,836 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 21:28:30,837 main DEBUG createAppenders(={Console}) 2024-11-20 21:28:30,838 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-20 21:28:30,838 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-20 21:28:30,838 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-20 21:28:30,839 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 21:28:30,839 main DEBUG OutputStream closed 2024-11-20 21:28:30,839 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 21:28:30,840 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 21:28:30,840 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-20 21:28:31,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 21:28:31,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 21:28:31,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 21:28:31,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 21:28:31,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 21:28:31,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 21:28:31,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 21:28:31,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 21:28:31,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 21:28:31,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 21:28:31,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 21:28:31,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 21:28:31,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 21:28:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 21:28:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 21:28:31,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 21:28:31,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 21:28:31,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 21:28:31,621 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 21:28:31,622 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-20 21:28:31,622 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 21:28:31,626 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-20T21:28:32,027 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad 2024-11-20 21:28:32,032 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 21:28:32,032 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T21:28:32,053 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-20T21:28:32,061 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-20T21:28:32,247 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T21:28:32,375 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T21:28:32,375 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T21:28:32,435 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T21:28:32,450 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59, deleteOnExit=true 2024-11-20T21:28:32,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T21:28:32,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/test.cache.data in system properties and HBase conf 2024-11-20T21:28:32,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T21:28:32,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir in system properties and HBase conf 2024-11-20T21:28:32,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T21:28:32,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T21:28:32,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T21:28:32,562 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T21:28:32,860 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:28:32,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T21:28:32,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T21:28:32,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T21:28:32,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T21:28:32,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T21:28:32,866 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T21:28:32,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T21:28:32,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T21:28:32,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T21:28:32,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/nfs.dump.dir in system properties and HBase conf 2024-11-20T21:28:32,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/java.io.tmpdir in system properties and HBase conf 2024-11-20T21:28:32,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T21:28:32,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T21:28:32,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T21:28:34,711 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T21:28:34,841 INFO [Time-limited test {}] log.Log(170): Logging initialized @5104ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T21:28:34,949 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:28:35,069 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:28:35,104 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:28:35,105 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:28:35,106 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T21:28:35,128 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:28:35,132 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a22ae35{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:28:35,133 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b2e6df8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:28:35,874 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5cbdf8c4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/java.io.tmpdir/jetty-localhost-42195-hadoop-hdfs-3_4_1-tests_jar-_-any-3312290432204420778/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T21:28:35,882 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@655f9c10{HTTP/1.1, (http/1.1)}{localhost:42195} 2024-11-20T21:28:35,883 INFO [Time-limited test {}] server.Server(415): Started @6146ms 2024-11-20T21:28:37,482 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:28:37,489 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:28:37,493 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:28:37,493 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:28:37,494 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T21:28:37,495 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@615a9ad9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:28:37,496 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ca6fdcd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:28:37,651 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c6d8c3d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/java.io.tmpdir/jetty-localhost-39573-hadoop-hdfs-3_4_1-tests_jar-_-any-4739688302202179954/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:28:37,660 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@56e06eb3{HTTP/1.1, (http/1.1)}{localhost:39573} 2024-11-20T21:28:37,660 INFO [Time-limited test {}] server.Server(415): Started @7923ms 2024-11-20T21:28:37,728 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T21:28:37,933 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:28:37,942 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:28:37,953 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:28:37,953 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:28:37,953 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T21:28:37,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b30c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:28:37,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2bea299b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:28:38,109 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5bccc2bb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/java.io.tmpdir/jetty-localhost-40309-hadoop-hdfs-3_4_1-tests_jar-_-any-1703429220731565145/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:28:38,110 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c0acdf3{HTTP/1.1, (http/1.1)}{localhost:40309} 2024-11-20T21:28:38,110 INFO [Time-limited test {}] server.Server(415): Started @8374ms 2024-11-20T21:28:38,113 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T21:28:38,177 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:28:38,182 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:28:38,187 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:28:38,187 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:28:38,187 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T21:28:38,188 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b0205f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:28:38,188 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e14f56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:28:38,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10e5d78d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/java.io.tmpdir/jetty-localhost-40163-hadoop-hdfs-3_4_1-tests_jar-_-any-7753004649942865338/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:28:38,348 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@148e0151{HTTP/1.1, (http/1.1)}{localhost:40163} 2024-11-20T21:28:38,349 INFO [Time-limited test {}] server.Server(415): Started @8612ms 2024-11-20T21:28:38,351 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T21:28:40,446 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data2/current/BP-422807794-172.17.0.3-1732138113638/current, will proceed with Du for space computation calculation, 2024-11-20T21:28:40,446 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data4/current/BP-422807794-172.17.0.3-1732138113638/current, will proceed with Du for space computation calculation, 2024-11-20T21:28:40,483 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data1/current/BP-422807794-172.17.0.3-1732138113638/current, will proceed with Du for space computation calculation, 2024-11-20T21:28:40,483 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data3/current/BP-422807794-172.17.0.3-1732138113638/current, will proceed with Du for space computation calculation, 2024-11-20T21:28:40,496 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data5/current/BP-422807794-172.17.0.3-1732138113638/current, will proceed with Du for space computation calculation, 2024-11-20T21:28:40,496 WARN [Thread-133 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data6/current/BP-422807794-172.17.0.3-1732138113638/current, will proceed with Du for space computation calculation, 2024-11-20T21:28:40,527 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T21:28:40,527 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T21:28:40,528 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T21:28:40,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97aa0e41770ed6a3 with lease ID 0xcaf6948fdf3d0240: Processing first storage report for DS-1079942e-6119-44ed-afe9-b5dcceee23cc from datanode DatanodeRegistration(127.0.0.1:38023, datanodeUuid=147809d0-3968-4205-9fd0-23adeea48b00, infoPort=34363, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638) 2024-11-20T21:28:40,580 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97aa0e41770ed6a3 with lease ID 0xcaf6948fdf3d0240: from storage DS-1079942e-6119-44ed-afe9-b5dcceee23cc node DatanodeRegistration(127.0.0.1:38023, datanodeUuid=147809d0-3968-4205-9fd0-23adeea48b00, infoPort=34363, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T21:28:40,580 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5aca501d96f2fa5 with lease ID 0xcaf6948fdf3d0242: Processing first storage report for DS-76880aaf-5eec-442b-b195-5d588f61907b from datanode DatanodeRegistration(127.0.0.1:46339, datanodeUuid=fb3d072d-f033-45f3-8fe2-0eabc36e7a26, infoPort=42083, infoSecurePort=0, ipcPort=42443, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638) 2024-11-20T21:28:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5aca501d96f2fa5 with lease ID 0xcaf6948fdf3d0242: from storage DS-76880aaf-5eec-442b-b195-5d588f61907b node DatanodeRegistration(127.0.0.1:46339, datanodeUuid=fb3d072d-f033-45f3-8fe2-0eabc36e7a26, infoPort=42083, infoSecurePort=0, ipcPort=42443, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:28:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d30bc0c4764c221 with lease ID 0xcaf6948fdf3d0241: Processing first storage report for DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c from datanode DatanodeRegistration(127.0.0.1:37559, datanodeUuid=db13ba50-1d12-4269-95db-acb57a9aa803, infoPort=39729, infoSecurePort=0, ipcPort=34105, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638) 2024-11-20T21:28:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d30bc0c4764c221 with lease ID 0xcaf6948fdf3d0241: from storage DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c node DatanodeRegistration(127.0.0.1:37559, datanodeUuid=db13ba50-1d12-4269-95db-acb57a9aa803, infoPort=39729, infoSecurePort=0, ipcPort=34105, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T21:28:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97aa0e41770ed6a3 with lease ID 0xcaf6948fdf3d0240: Processing first storage report for DS-091744ca-ee22-43cf-9cef-79f7e3d08d90 from datanode DatanodeRegistration(127.0.0.1:38023, datanodeUuid=147809d0-3968-4205-9fd0-23adeea48b00, infoPort=34363, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638) 2024-11-20T21:28:40,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97aa0e41770ed6a3 with lease ID 0xcaf6948fdf3d0240: from storage DS-091744ca-ee22-43cf-9cef-79f7e3d08d90 node DatanodeRegistration(127.0.0.1:38023, datanodeUuid=147809d0-3968-4205-9fd0-23adeea48b00, infoPort=34363, infoSecurePort=0, ipcPort=46269, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:28:40,582 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5aca501d96f2fa5 with lease ID 0xcaf6948fdf3d0242: Processing first storage report for DS-3c21bb78-54a8-4ef5-a77d-a365e445e450 from datanode DatanodeRegistration(127.0.0.1:46339, datanodeUuid=fb3d072d-f033-45f3-8fe2-0eabc36e7a26, infoPort=42083, infoSecurePort=0, ipcPort=42443, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638) 2024-11-20T21:28:40,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5aca501d96f2fa5 with lease ID 0xcaf6948fdf3d0242: from storage DS-3c21bb78-54a8-4ef5-a77d-a365e445e450 node DatanodeRegistration(127.0.0.1:46339, datanodeUuid=fb3d072d-f033-45f3-8fe2-0eabc36e7a26, infoPort=42083, infoSecurePort=0, ipcPort=42443, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:28:40,582 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d30bc0c4764c221 with lease ID 0xcaf6948fdf3d0241: Processing first storage report for DS-417e5593-5af8-4c28-b7fc-db8253be08d7 from datanode DatanodeRegistration(127.0.0.1:37559, datanodeUuid=db13ba50-1d12-4269-95db-acb57a9aa803, infoPort=39729, infoSecurePort=0, ipcPort=34105, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638) 2024-11-20T21:28:40,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d30bc0c4764c221 with lease ID 0xcaf6948fdf3d0241: from storage DS-417e5593-5af8-4c28-b7fc-db8253be08d7 node DatanodeRegistration(127.0.0.1:37559, datanodeUuid=db13ba50-1d12-4269-95db-acb57a9aa803, infoPort=39729, infoSecurePort=0, ipcPort=34105, storageInfo=lv=-57;cid=testClusterID;nsid=1580195918;c=1732138113638), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:28:40,634 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad 2024-11-20T21:28:40,730 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/zookeeper_0, clientPort=56682, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T21:28:40,741 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56682 2024-11-20T21:28:40,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:40,770 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:40,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741825_1001 (size=7) 2024-11-20T21:28:40,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741825_1001 (size=7) 2024-11-20T21:28:40,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741825_1001 (size=7) 2024-11-20T21:28:41,399 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c with version=8 2024-11-20T21:28:41,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/hbase-staging 2024-11-20T21:28:41,804 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0b16ab3f13b7:0 server-side Connection retries=45 2024-11-20T21:28:41,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:41,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:41,825 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:28:41,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:41,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:28:41,989 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T21:28:42,062 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T21:28:42,073 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T21:28:42,078 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:28:42,113 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 18371 (auto-detected) 2024-11-20T21:28:42,114 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-20T21:28:42,139 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39535 2024-11-20T21:28:42,160 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39535 connecting to ZooKeeper ensemble=127.0.0.1:56682 2024-11-20T21:28:42,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:395350x0, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:28:42,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39535-0x1005f1840a00000 connected 2024-11-20T21:28:42,610 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:42,615 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:42,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:28:42,636 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c, hbase.cluster.distributed=false 2024-11-20T21:28:42,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:28:42,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39535 2024-11-20T21:28:42,668 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39535 2024-11-20T21:28:42,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39535 2024-11-20T21:28:42,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39535 2024-11-20T21:28:42,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39535 2024-11-20T21:28:42,799 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b16ab3f13b7:0 server-side Connection retries=45 2024-11-20T21:28:42,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:42,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:42,803 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:28:42,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:42,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:28:42,807 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:28:42,809 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:28:42,810 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33007 2024-11-20T21:28:42,813 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33007 connecting to ZooKeeper ensemble=127.0.0.1:56682 2024-11-20T21:28:42,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:42,817 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:42,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330070x0, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:28:42,854 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33007-0x1005f1840a00001 connected 2024-11-20T21:28:42,854 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:28:42,859 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:28:42,869 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:28:42,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:28:42,879 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:28:42,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33007 2024-11-20T21:28:42,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33007 2024-11-20T21:28:42,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33007 2024-11-20T21:28:42,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33007 2024-11-20T21:28:42,889 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33007 2024-11-20T21:28:42,913 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b16ab3f13b7:0 server-side Connection retries=45 2024-11-20T21:28:42,913 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:42,914 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:42,914 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:28:42,914 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:42,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:28:42,915 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:28:42,915 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:28:42,917 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45827 2024-11-20T21:28:42,920 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45827 connecting to ZooKeeper ensemble=127.0.0.1:56682 2024-11-20T21:28:42,921 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:42,925 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:42,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:458270x0, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:28:42,955 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:28:42,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458270x0, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:28:42,961 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45827-0x1005f1840a00002 connected 2024-11-20T21:28:42,964 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:28:42,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:28:42,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:28:42,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45827 2024-11-20T21:28:42,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45827 2024-11-20T21:28:42,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45827 2024-11-20T21:28:42,983 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45827 2024-11-20T21:28:42,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45827 2024-11-20T21:28:43,010 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b16ab3f13b7:0 server-side Connection retries=45 2024-11-20T21:28:43,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:43,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:43,011 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:28:43,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:28:43,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:28:43,011 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:28:43,012 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:28:43,039 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40171 2024-11-20T21:28:43,041 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40171 connecting to ZooKeeper ensemble=127.0.0.1:56682 2024-11-20T21:28:43,042 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:43,045 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:43,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401710x0, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:28:43,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:401710x0, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:28:43,101 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40171-0x1005f1840a00003 connected 2024-11-20T21:28:43,102 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:28:43,103 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:28:43,104 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:28:43,106 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:28:43,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40171 2024-11-20T21:28:43,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40171 2024-11-20T21:28:43,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40171 2024-11-20T21:28:43,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40171 2024-11-20T21:28:43,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40171 2024-11-20T21:28:43,128 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b16ab3f13b7:39535 2024-11-20T21:28:43,129 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:43,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,171 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:43,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:28:43,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:28:43,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:28:43,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,238 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T21:28:43,240 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b16ab3f13b7,39535,1732138121558 from backup master directory 2024-11-20T21:28:43,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:43,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:28:43,277 WARN [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:28:43,278 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:43,280 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T21:28:43,281 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T21:28:43,343 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/hbase.id] with ID: e28df62c-830d-4222-9e72-1212682cc572 2024-11-20T21:28:43,343 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/.tmp/hbase.id 2024-11-20T21:28:43,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741826_1002 (size=42) 2024-11-20T21:28:43,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741826_1002 (size=42) 2024-11-20T21:28:43,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741826_1002 (size=42) 2024-11-20T21:28:43,361 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/.tmp/hbase.id]:[hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/hbase.id] 2024-11-20T21:28:43,406 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:28:43,410 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T21:28:43,435 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-20T21:28:43,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:43,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741827_1003 (size=196) 2024-11-20T21:28:43,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741827_1003 (size=196) 2024-11-20T21:28:43,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741827_1003 (size=196) 2024-11-20T21:28:43,519 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:28:43,521 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T21:28:43,528 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:43,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741828_1004 (size=1189) 2024-11-20T21:28:43,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741828_1004 (size=1189) 2024-11-20T21:28:43,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741828_1004 (size=1189) 2024-11-20T21:28:43,585 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store 2024-11-20T21:28:43,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741829_1005 (size=34) 2024-11-20T21:28:43,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741829_1005 (size=34) 2024-11-20T21:28:43,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741829_1005 (size=34) 2024-11-20T21:28:43,612 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T21:28:43,615 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:43,617 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:28:43,617 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:28:43,618 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:28:43,619 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:28:43,620 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:28:43,620 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:28:43,621 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732138123617Disabling compacts and flushes for region at 1732138123617Disabling writes for close at 1732138123620 (+3 ms)Writing region close event to WAL at 1732138123620Closed at 1732138123620 2024-11-20T21:28:43,623 WARN [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/.initializing 2024-11-20T21:28:43,624 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/WALs/0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:43,633 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:43,652 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C39535%2C1732138121558, suffix=, logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/WALs/0b16ab3f13b7,39535,1732138121558, archiveDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/oldWALs, maxLogs=10 2024-11-20T21:28:43,690 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/WALs/0b16ab3f13b7,39535,1732138121558/0b16ab3f13b7%2C39535%2C1732138121558.1732138123657, exclude list is [], retry=0 2024-11-20T21:28:43,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:28:43,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:43,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:43,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:43,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T21:28:43,769 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-20T21:28:43,877 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/WALs/0b16ab3f13b7,39535,1732138121558/0b16ab3f13b7%2C39535%2C1732138121558.1732138123657 2024-11-20T21:28:43,879 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:28:43,879 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:28:43,880 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:43,884 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:43,884 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:43,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:43,951 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T21:28:43,956 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:43,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:43,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:43,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T21:28:43,967 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:43,983 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:43,983 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:43,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T21:28:43,992 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:43,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:43,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:43,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T21:28:43,998 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:44,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:44,006 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:44,011 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:44,013 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:44,020 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:44,020 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:44,033 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:28:44,039 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:28:44,062 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:44,063 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67857572, jitterRate=0.011156618595123291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:28:44,070 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732138123896Initializing all the Stores at 1732138123899 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138123899Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138123900 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138123900Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138123900Cleaning up temporary data from old regions at 1732138124021 (+121 ms)Region opened successfully at 1732138124070 (+49 ms) 2024-11-20T21:28:44,078 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T21:28:44,134 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d82d488, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b16ab3f13b7/172.17.0.3:0 2024-11-20T21:28:44,183 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T21:28:44,199 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T21:28:44,201 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T21:28:44,205 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T21:28:44,207 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T21:28:44,216 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-20T21:28:44,216 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T21:28:44,251 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T21:28:44,260 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T21:28:44,306 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T21:28:44,309 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T21:28:44,310 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T21:28:44,344 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T21:28:44,347 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T21:28:44,351 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T21:28:44,375 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T21:28:44,376 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T21:28:44,405 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T21:28:44,430 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T21:28:44,460 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T21:28:44,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:28:44,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:28:44,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:28:44,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:28:44,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,508 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b16ab3f13b7,39535,1732138121558, sessionid=0x1005f1840a00000, setting cluster-up flag (Was=false) 2024-11-20T21:28:44,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,821 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T21:28:44,824 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:44,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:44,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:45,029 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T21:28:45,031 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:45,040 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T21:28:45,116 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T21:28:45,120 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(746): ClusterId : e28df62c-830d-4222-9e72-1212682cc572 2024-11-20T21:28:45,121 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(746): ClusterId : e28df62c-830d-4222-9e72-1212682cc572 2024-11-20T21:28:45,123 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(746): ClusterId : e28df62c-830d-4222-9e72-1212682cc572 2024-11-20T21:28:45,124 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:28:45,125 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:28:45,126 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:28:45,129 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T21:28:45,136 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T21:28:45,142 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b16ab3f13b7,39535,1732138121558 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T21:28:45,218 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:28:45,218 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:28:45,218 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:28:45,218 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:28:45,218 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:28:45,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:28:45,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:28:45,218 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:28:45,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:28:45,219 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:28:45,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b16ab3f13b7:0, corePoolSize=10, maxPoolSize=10 2024-11-20T21:28:45,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,220 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:28:45,220 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,226 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732138155226 2024-11-20T21:28:45,227 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:28:45,228 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T21:28:45,228 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T21:28:45,229 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T21:28:45,233 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T21:28:45,233 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T21:28:45,234 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T21:28:45,234 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T21:28:45,236 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:45,237 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T21:28:45,237 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,241 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T21:28:45,242 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T21:28:45,243 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T21:28:45,249 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T21:28:45,249 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T21:28:45,252 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.large.0-1732138125251,5,FailOnTimeoutGroup] 2024-11-20T21:28:45,253 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.small.0-1732138125253,5,FailOnTimeoutGroup] 2024-11-20T21:28:45,253 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,253 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T21:28:45,255 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,255 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741831_1007 (size=1321) 2024-11-20T21:28:45,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741831_1007 (size=1321) 2024-11-20T21:28:45,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741831_1007 (size=1321) 2024-11-20T21:28:45,277 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T21:28:45,277 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c 2024-11-20T21:28:45,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741832_1008 (size=32) 2024-11-20T21:28:45,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741832_1008 (size=32) 2024-11-20T21:28:45,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741832_1008 (size=32) 2024-11-20T21:28:45,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:45,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T21:28:45,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:28:45,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:45,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:45,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T21:28:45,317 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:28:45,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:45,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:45,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T21:28:45,321 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:28:45,321 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:45,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:45,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T21:28:45,324 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:28:45,324 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:45,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:45,325 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:28:45,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740 2024-11-20T21:28:45,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740 2024-11-20T21:28:45,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:28:45,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:28:45,332 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:28:45,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:28:45,339 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:28:45,339 DEBUG [RS:1;0b16ab3f13b7:45827 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b527c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b16ab3f13b7/172.17.0.3:0 2024-11-20T21:28:45,349 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:28:45,349 DEBUG [RS:2;0b16ab3f13b7:40171 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14068401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b16ab3f13b7/172.17.0.3:0 2024-11-20T21:28:45,362 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:28:45,362 DEBUG [RS:0;0b16ab3f13b7:33007 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d03a6dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b16ab3f13b7/172.17.0.3:0 2024-11-20T21:28:45,364 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:45,366 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68682057, jitterRate=0.023442402482032776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:28:45,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732138125308Initializing all the Stores at 1732138125310 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138125310Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138125310Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138125310Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138125310Cleaning up temporary data from old regions at 1732138125331 (+21 ms)Region opened successfully at 1732138125370 (+39 ms) 2024-11-20T21:28:45,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:28:45,370 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:28:45,370 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:28:45,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:28:45,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:28:45,373 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0b16ab3f13b7:40171 2024-11-20T21:28:45,377 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:28:45,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732138125370Disabling compacts and flushes for region at 1732138125370Disabling writes for close at 1732138125371 (+1 ms)Writing region close event to WAL at 1732138125376 (+5 ms)Closed at 1732138125377 (+1 ms) 2024-11-20T21:28:45,378 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:28:45,378 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:28:45,378 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:28:45,382 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0b16ab3f13b7:45827 2024-11-20T21:28:45,382 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:28:45,382 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:28:45,383 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:28:45,383 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:28:45,383 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T21:28:45,388 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b16ab3f13b7:33007 2024-11-20T21:28:45,388 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:28:45,388 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:28:45,388 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:28:45,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T21:28:45,393 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b16ab3f13b7,39535,1732138121558 with port=45827, startcode=1732138122912 2024-11-20T21:28:45,393 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b16ab3f13b7,39535,1732138121558 with port=40171, startcode=1732138123009 2024-11-20T21:28:45,393 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b16ab3f13b7,39535,1732138121558 with port=33007, startcode=1732138122755 2024-11-20T21:28:45,405 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:28:45,411 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T21:28:45,417 DEBUG [RS:2;0b16ab3f13b7:40171 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:28:45,417 DEBUG [RS:1;0b16ab3f13b7:45827 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:28:45,420 DEBUG [RS:0;0b16ab3f13b7:33007 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:28:45,461 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43117, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:28:45,461 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33815, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:28:45,461 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43213, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:28:45,468 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b16ab3f13b7,45827,1732138122912 2024-11-20T21:28:45,472 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(517): Registering regionserver=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:28:45,487 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:45,487 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(517): Registering regionserver=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:45,491 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c 2024-11-20T21:28:45,491 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44823 2024-11-20T21:28:45,491 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:28:45,494 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c 2024-11-20T21:28:45,494 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b16ab3f13b7,40171,1732138123009 2024-11-20T21:28:45,494 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44823 2024-11-20T21:28:45,494 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:28:45,494 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(517): Registering regionserver=0b16ab3f13b7,40171,1732138123009 2024-11-20T21:28:45,498 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c 2024-11-20T21:28:45,498 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44823 2024-11-20T21:28:45,498 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:28:45,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:28:45,563 WARN [0b16ab3f13b7:39535 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T21:28:45,699 DEBUG [RS:1;0b16ab3f13b7:45827 {}] zookeeper.ZKUtil(111): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b16ab3f13b7,45827,1732138122912 2024-11-20T21:28:45,699 WARN [RS:1;0b16ab3f13b7:45827 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:28:45,699 INFO [RS:1;0b16ab3f13b7:45827 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:45,700 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,45827,1732138122912 2024-11-20T21:28:45,700 DEBUG [RS:2;0b16ab3f13b7:40171 {}] zookeeper.ZKUtil(111): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b16ab3f13b7,40171,1732138123009 2024-11-20T21:28:45,700 DEBUG [RS:0;0b16ab3f13b7:33007 {}] zookeeper.ZKUtil(111): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:45,700 WARN [RS:2;0b16ab3f13b7:40171 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:28:45,700 WARN [RS:0;0b16ab3f13b7:33007 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:28:45,700 INFO [RS:2;0b16ab3f13b7:40171 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:45,700 INFO [RS:0;0b16ab3f13b7:33007 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:45,701 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:45,701 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,40171,1732138123009 2024-11-20T21:28:45,701 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b16ab3f13b7,33007,1732138122755] 2024-11-20T21:28:45,701 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b16ab3f13b7,40171,1732138123009] 2024-11-20T21:28:45,702 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b16ab3f13b7,45827,1732138122912] 2024-11-20T21:28:45,732 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:28:45,732 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:28:45,732 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:28:45,750 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:28:45,750 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:28:45,752 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:28:45,757 INFO [RS:1;0b16ab3f13b7:45827 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:28:45,757 INFO [RS:2;0b16ab3f13b7:40171 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:28:45,757 INFO [RS:0;0b16ab3f13b7:33007 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:28:45,757 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,757 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,757 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,760 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:28:45,761 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:28:45,764 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:28:45,766 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:28:45,766 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:28:45,768 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:28:45,768 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,768 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,768 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,768 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,768 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,768 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:28:45,769 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:28:45,769 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:28:45,769 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,769 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,770 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:28:45,771 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:28:45,771 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:28:45,771 DEBUG [RS:2;0b16ab3f13b7:40171 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:28:45,771 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:28:45,771 DEBUG [RS:0;0b16ab3f13b7:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:28:45,771 DEBUG [RS:1;0b16ab3f13b7:45827 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:28:45,772 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,773 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,773 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,773 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,773 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,773 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,33007,1732138122755-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:28:45,781 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,781 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,781 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,781 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,781 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,782 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,45827,1732138122912-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:28:45,784 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,785 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,785 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,785 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,785 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,785 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,40171,1732138123009-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:28:45,801 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:28:45,803 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,33007,1732138122755-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,804 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,804 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.Replication(171): 0b16ab3f13b7,33007,1732138122755 started 2024-11-20T21:28:45,809 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:28:45,809 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,45827,1732138122912-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,809 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,810 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.Replication(171): 0b16ab3f13b7,45827,1732138122912 started 2024-11-20T21:28:45,819 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:28:45,819 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,40171,1732138123009-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,819 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,820 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.Replication(171): 0b16ab3f13b7,40171,1732138123009 started 2024-11-20T21:28:45,829 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,830 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1482): Serving as 0b16ab3f13b7,33007,1732138122755, RpcServer on 0b16ab3f13b7/172.17.0.3:33007, sessionid=0x1005f1840a00001 2024-11-20T21:28:45,831 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:28:45,831 DEBUG [RS:0;0b16ab3f13b7:33007 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:45,831 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,33007,1732138122755' 2024-11-20T21:28:45,831 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:28:45,832 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,832 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1482): Serving as 0b16ab3f13b7,45827,1732138122912, RpcServer on 0b16ab3f13b7/172.17.0.3:45827, sessionid=0x1005f1840a00002 2024-11-20T21:28:45,833 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:28:45,833 DEBUG [RS:1;0b16ab3f13b7:45827 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b16ab3f13b7,45827,1732138122912 2024-11-20T21:28:45,833 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:28:45,833 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,45827,1732138122912' 2024-11-20T21:28:45,833 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:28:45,834 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:28:45,834 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:28:45,834 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:28:45,834 DEBUG [RS:0;0b16ab3f13b7:33007 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:45,834 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,33007,1732138122755' 2024-11-20T21:28:45,834 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:28:45,835 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:28:45,835 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:28:45,835 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:28:45,835 DEBUG [RS:1;0b16ab3f13b7:45827 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b16ab3f13b7,45827,1732138122912 2024-11-20T21:28:45,835 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,45827,1732138122912' 2024-11-20T21:28:45,835 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:28:45,836 DEBUG [RS:0;0b16ab3f13b7:33007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:28:45,836 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:28:45,836 INFO [RS:0;0b16ab3f13b7:33007 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:28:45,836 INFO [RS:0;0b16ab3f13b7:33007 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:28:45,837 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:45,837 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1482): Serving as 0b16ab3f13b7,40171,1732138123009, RpcServer on 0b16ab3f13b7/172.17.0.3:40171, sessionid=0x1005f1840a00003 2024-11-20T21:28:45,837 DEBUG [RS:1;0b16ab3f13b7:45827 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:28:45,837 INFO [RS:1;0b16ab3f13b7:45827 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:28:45,837 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:28:45,837 INFO [RS:1;0b16ab3f13b7:45827 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:28:45,837 DEBUG [RS:2;0b16ab3f13b7:40171 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b16ab3f13b7,40171,1732138123009 2024-11-20T21:28:45,837 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,40171,1732138123009' 2024-11-20T21:28:45,837 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:28:45,838 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:28:45,838 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:28:45,838 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:28:45,839 DEBUG [RS:2;0b16ab3f13b7:40171 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b16ab3f13b7,40171,1732138123009 2024-11-20T21:28:45,839 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,40171,1732138123009' 2024-11-20T21:28:45,839 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:28:45,839 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:28:45,840 DEBUG [RS:2;0b16ab3f13b7:40171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:28:45,840 INFO [RS:2;0b16ab3f13b7:40171 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:28:45,840 INFO [RS:2;0b16ab3f13b7:40171 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:28:45,942 INFO [RS:2;0b16ab3f13b7:40171 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:45,942 INFO [RS:0;0b16ab3f13b7:33007 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:45,942 INFO [RS:1;0b16ab3f13b7:45827 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:45,945 INFO [RS:0;0b16ab3f13b7:33007 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C33007%2C1732138122755, suffix=, logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755, archiveDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs, maxLogs=32 2024-11-20T21:28:45,945 INFO [RS:1;0b16ab3f13b7:45827 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C45827%2C1732138122912, suffix=, logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,45827,1732138122912, archiveDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs, maxLogs=32 2024-11-20T21:28:45,945 INFO [RS:2;0b16ab3f13b7:40171 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C40171%2C1732138123009, suffix=, logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,40171,1732138123009, archiveDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs, maxLogs=32 2024-11-20T21:28:45,964 DEBUG [RS:1;0b16ab3f13b7:45827 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,45827,1732138122912/0b16ab3f13b7%2C45827%2C1732138122912.1732138125950, exclude list is [], retry=0 2024-11-20T21:28:45,967 DEBUG [RS:0;0b16ab3f13b7:33007 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, exclude list is [], retry=0 2024-11-20T21:28:45,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:45,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:45,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:45,972 DEBUG [RS:2;0b16ab3f13b7:40171 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,40171,1732138123009/0b16ab3f13b7%2C40171%2C1732138123009.1732138125950, exclude list is [], retry=0 2024-11-20T21:28:45,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:45,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:45,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:45,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:45,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:45,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:46,023 INFO [RS:0;0b16ab3f13b7:33007 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 2024-11-20T21:28:46,023 INFO [RS:1;0b16ab3f13b7:45827 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,45827,1732138122912/0b16ab3f13b7%2C45827%2C1732138122912.1732138125950 2024-11-20T21:28:46,024 DEBUG [RS:0;0b16ab3f13b7:33007 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729)] 2024-11-20T21:28:46,024 DEBUG [RS:1;0b16ab3f13b7:45827 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:46,029 INFO [RS:2;0b16ab3f13b7:40171 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,40171,1732138123009/0b16ab3f13b7%2C40171%2C1732138123009.1732138125950 2024-11-20T21:28:46,030 DEBUG [RS:2;0b16ab3f13b7:40171 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:46,068 DEBUG [0b16ab3f13b7:39535 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T21:28:46,079 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(204): Hosts are {0b16ab3f13b7=0} racks are {/default-rack=0} 2024-11-20T21:28:46,087 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:28:46,087 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:28:46,087 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T21:28:46,087 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:28:46,087 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:28:46,087 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T21:28:46,087 INFO [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:28:46,088 INFO [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:28:46,088 INFO [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T21:28:46,088 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:28:46,095 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:46,101 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b16ab3f13b7,33007,1732138122755, state=OPENING 2024-11-20T21:28:46,136 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T21:28:46,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:46,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:46,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:46,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:28:46,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,178 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:28:46,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,33007,1732138122755}] 2024-11-20T21:28:46,360 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:28:46,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45867, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:28:46,375 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T21:28:46,376 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:46,376 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T21:28:46,379 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C33007%2C1732138122755.meta, suffix=.meta, logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755, archiveDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs, maxLogs=32 2024-11-20T21:28:46,396 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, exclude list is [], retry=0 2024-11-20T21:28:46,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:46,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:46,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:46,482 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta 2024-11-20T21:28:46,482 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:46,482 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:28:46,484 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T21:28:46,487 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T21:28:46,492 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T21:28:46,496 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T21:28:46,497 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:46,497 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T21:28:46,497 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T21:28:46,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T21:28:46,501 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:28:46,501 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:46,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:46,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T21:28:46,504 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:28:46,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:46,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:46,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T21:28:46,506 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:28:46,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:46,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:46,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T21:28:46,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:28:46,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:46,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:28:46,509 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:28:46,510 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740 2024-11-20T21:28:46,513 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740 2024-11-20T21:28:46,515 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:28:46,515 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:28:46,516 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:28:46,519 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:28:46,520 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72711476, jitterRate=0.0834854245185852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:28:46,520 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T21:28:46,522 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732138126497Writing region info on filesystem at 1732138126498 (+1 ms)Initializing all the Stores at 1732138126499 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138126499Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138126500 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138126500Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138126500Cleaning up temporary data from old regions at 1732138126515 (+15 ms)Running coprocessor post-open hooks at 1732138126520 (+5 ms)Region opened successfully at 1732138126521 (+1 ms) 2024-11-20T21:28:46,529 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732138126351 2024-11-20T21:28:46,540 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T21:28:46,540 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T21:28:46,543 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:46,545 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b16ab3f13b7,33007,1732138122755, state=OPEN 2024-11-20T21:28:46,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:28:46,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:28:46,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:28:46,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:28:46,590 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,591 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:28:46,591 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:28:46,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T21:28:46,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,33007,1732138122755 in 410 msec 2024-11-20T21:28:46,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T21:28:46,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.2400 sec 2024-11-20T21:28:46,638 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:28:46,638 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T21:28:46,680 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:28:46,682 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1] 2024-11-20T21:28:46,705 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:28:46,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56943, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:28:46,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6630 sec 2024-11-20T21:28:46,733 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732138126733, completionTime=-1 2024-11-20T21:28:46,736 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T21:28:46,736 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T21:28:46,805 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-20T21:28:46,806 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732138186806 2024-11-20T21:28:46,806 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732138246806 2024-11-20T21:28:46,806 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 69 msec 2024-11-20T21:28:46,808 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T21:28:46,822 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39535,1732138121558-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:46,824 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39535,1732138121558-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:46,824 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39535,1732138121558-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:46,831 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b16ab3f13b7:39535, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:46,831 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:46,832 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:46,848 DEBUG [master/0b16ab3f13b7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T21:28:46,876 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.598sec 2024-11-20T21:28:46,878 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T21:28:46,879 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T21:28:46,880 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T21:28:46,881 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T21:28:46,881 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T21:28:46,882 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39535,1732138121558-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:28:46,882 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39535,1732138121558-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T21:28:46,887 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T21:28:46,888 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T21:28:46,888 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39535,1732138121558-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:28:46,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dff5e3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:28:46,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39535,-1 for getting cluster id 2024-11-20T21:28:46,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:28:46,949 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e28df62c-830d-4222-9e72-1212682cc572' 2024-11-20T21:28:46,951 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:28:46,951 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e28df62c-830d-4222-9e72-1212682cc572" 2024-11-20T21:28:46,952 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5274a1b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:28:46,952 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39535,-1] 2024-11-20T21:28:46,954 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:28:46,956 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:28:46,958 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:28:46,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514f4ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:28:46,965 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:28:46,972 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1] 2024-11-20T21:28:46,973 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:28:46,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:28:46,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b16ab3f13b7,39535,1732138121558 2024-11-20T21:28:46,996 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44823/hbase 2024-11-20T21:28:47,023 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=641, ProcessCount=11, AvailableMemoryMB=3689 2024-11-20T21:28:47,050 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:47,054 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:47,055 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:47,060 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95931171, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-95931171, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:47,086 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95931171/hregion-95931171.1732138127066, exclude list is [], retry=0 2024-11-20T21:28:47,091 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:47,092 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:47,096 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:47,147 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95931171/hregion-95931171.1732138127066 2024-11-20T21:28:47,150 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:28:47,150 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 25737f55837714bd017e2a12c7a32804, NAME => 'testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:28:47,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741838_1014 (size=64) 2024-11-20T21:28:47,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741838_1014 (size=64) 2024-11-20T21:28:47,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741838_1014 (size=64) 2024-11-20T21:28:47,206 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:47,211 INFO [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,215 INFO [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25737f55837714bd017e2a12c7a32804 columnFamilyName a 2024-11-20T21:28:47,215 DEBUG [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:47,216 INFO [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] regionserver.HStore(327): Store=25737f55837714bd017e2a12c7a32804/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:47,216 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,220 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,221 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,221 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,225 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,234 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:47,235 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 25737f55837714bd017e2a12c7a32804; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63164648, jitterRate=-0.05877339839935303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:28:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 25737f55837714bd017e2a12c7a32804: Writing region info on filesystem at 1732138127206Initializing all the Stores at 1732138127208 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138127208Cleaning up temporary data from old regions at 1732138127221 (+13 ms)Region opened successfully at 1732138127237 (+16 ms) 2024-11-20T21:28:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 25737f55837714bd017e2a12c7a32804, disabling compactions & flushes 2024-11-20T21:28:47,237 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804. 2024-11-20T21:28:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804. 2024-11-20T21:28:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804. after waiting 0 ms 2024-11-20T21:28:47,237 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804. 2024-11-20T21:28:47,238 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804. 2024-11-20T21:28:47,238 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 25737f55837714bd017e2a12c7a32804: Waiting for close lock at 1732138127237Disabling compacts and flushes for region at 1732138127237Disabling writes for close at 1732138127237Writing region close event to WAL at 1732138127238 (+1 ms)Closed at 1732138127238 2024-11-20T21:28:47,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741837_1013 (size=175) 2024-11-20T21:28:47,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741837_1013 (size=175) 2024-11-20T21:28:47,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741837_1013 (size=175) 2024-11-20T21:28:47,261 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:28:47,261 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95931171:(num 1732138127066) 2024-11-20T21:28:47,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T21:28:47,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741839_1015 (size=469) 2024-11-20T21:28:47,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741839_1015 (size=469) 2024-11-20T21:28:47,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741839_1015 (size=469) 2024-11-20T21:28:47,375 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T21:28:47,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741840_1016 (size=322) 2024-11-20T21:28:47,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741840_1016 (size=322) 2024-11-20T21:28:47,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741840_1016 (size=322) 2024-11-20T21:28:47,472 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1, size=469 (469bytes) 2024-11-20T21:28:47,472 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T21:28:47,473 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T21:28:47,473 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1 2024-11-20T21:28:47,479 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1 after 4ms 2024-11-20T21:28:47,534 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:47,535 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1 took 65ms 2024-11-20T21:28:47,547 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1 so closing down 2024-11-20T21:28:47,547 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:47,551 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T21:28:47,585 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T21:28:47,587 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741841_1017 (size=469) 2024-11-20T21:28:47,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741841_1017 (size=469) 2024-11-20T21:28:47,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741841_1017 (size=469) 2024-11-20T21:28:47,606 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T21:28:47,610 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002 2024-11-20T21:28:47,615 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-20T21:28:47,615 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1, journal: Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1, size=469 (469bytes) at 1732138127472Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1 so closing down at 1732138127547 (+75 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000001-wal-1.temp at 1732138127585 (+38 ms)3 split writer threads finished at 1732138127587 (+2 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732138127606 (+19 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002 at 1732138127610 (+4 ms)Processed 2 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732138127615 (+5 ms) 2024-11-20T21:28:47,638 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2, size=322 (322bytes) 2024-11-20T21:28:47,638 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2 2024-11-20T21:28:47,640 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2 after 2ms 2024-11-20T21:28:47,680 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:47,680 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2 took 42ms 2024-11-20T21:28:47,683 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2 so closing down 2024-11-20T21:28:47,683 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:47,686 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T21:28:47,717 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T21:28:47,717 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:47,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741842_1018 (size=322) 2024-11-20T21:28:47,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741842_1018 (size=322) 2024-11-20T21:28:47,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741842_1018 (size=322) 2024-11-20T21:28:47,738 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T21:28:47,775 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:47,779 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-20T21:28:47,781 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 101 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-20T21:28:47,781 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2, journal: Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2, size=322 (322bytes) at 1732138127638Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2 so closing down at 1732138127683 (+45 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002-wal-2.temp at 1732138127717 (+34 ms)3 split writer threads finished at 1732138127717Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732138127738 (+21 ms)Processed 1 edits across 1 Regions in 101 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732138127781 (+43 ms) 2024-11-20T21:28:47,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:47,785 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:47,804 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal.1732138127786, exclude list is [], retry=0 2024-11-20T21:28:47,808 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:47,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:47,810 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:47,853 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal.1732138127786 2024-11-20T21:28:47,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:28:47,860 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 25737f55837714bd017e2a12c7a32804, NAME => 'testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:28:47,861 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:47,861 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,861 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,864 INFO [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,865 INFO [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25737f55837714bd017e2a12c7a32804 columnFamilyName a 2024-11-20T21:28:47,866 DEBUG [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:47,867 INFO [StoreOpener-25737f55837714bd017e2a12c7a32804-1 {}] regionserver.HStore(327): Store=25737f55837714bd017e2a12c7a32804/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:47,867 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,872 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:47,873 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002 2024-11-20T21:28:47,906 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:47,914 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002 2024-11-20T21:28:47,918 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 25737f55837714bd017e2a12c7a32804 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T21:28:47,985 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/.tmp/a/2699020cd44944d4bfcb9ab60a47777c is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732138127261/Put/seqid=0 2024-11-20T21:28:48,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741844_1020 (size=5170) 2024-11-20T21:28:48,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741844_1020 (size=5170) 2024-11-20T21:28:48,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741844_1020 (size=5170) 2024-11-20T21:28:48,007 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/.tmp/a/2699020cd44944d4bfcb9ab60a47777c 2024-11-20T21:28:48,057 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/.tmp/a/2699020cd44944d4bfcb9ab60a47777c as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/a/2699020cd44944d4bfcb9ab60a47777c 2024-11-20T21:28:48,069 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/a/2699020cd44944d4bfcb9ab60a47777c, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T21:28:48,078 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 25737f55837714bd017e2a12c7a32804 in 158ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T21:28:48,080 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/0000000000000000002 2024-11-20T21:28:48,081 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:48,081 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:48,088 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 25737f55837714bd017e2a12c7a32804 2024-11-20T21:28:48,091 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/25737f55837714bd017e2a12c7a32804/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T21:28:48,093 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 25737f55837714bd017e2a12c7a32804; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71120410, jitterRate=0.05977669358253479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:28:48,095 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 25737f55837714bd017e2a12c7a32804: Writing region info on filesystem at 1732138127861Initializing all the Stores at 1732138127863 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138127863Obtaining lock to block concurrent updates at 1732138127918 (+55 ms)Preparing flush snapshotting stores in 25737f55837714bd017e2a12c7a32804 at 1732138127919 (+1 ms)Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732138127922 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732138127051.25737f55837714bd017e2a12c7a32804. at 1732138127922Flushing 25737f55837714bd017e2a12c7a32804/a: creating writer at 1732138127924 (+2 ms)Flushing 25737f55837714bd017e2a12c7a32804/a: appending metadata at 1732138127971 (+47 ms)Flushing 25737f55837714bd017e2a12c7a32804/a: closing flushed file at 1732138127975 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@208fbf54: reopening flushed file at 1732138128055 (+80 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 25737f55837714bd017e2a12c7a32804 in 158ms, sequenceid=2, compaction requested=false; wal=null at 1732138128078 (+23 ms)Cleaning up temporary data from old regions at 1732138128081 (+3 ms)Region opened successfully at 1732138128095 (+14 ms) 2024-11-20T21:28:48,124 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=375 (was 363) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:32784 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:52624 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:32790 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57102 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57068 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44823/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=641 (was 641), ProcessCount=11 (was 11), AvailableMemoryMB=3532 (was 3689) 2024-11-20T21:28:48,137 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=641, ProcessCount=11, AvailableMemoryMB=3531 2024-11-20T21:28:48,154 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:48,157 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:48,158 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:48,177 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85291737, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-85291737, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:48,194 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85291737/hregion-85291737.1732138128178, exclude list is [], retry=0 2024-11-20T21:28:48,201 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:48,205 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:48,205 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:48,243 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85291737/hregion-85291737.1732138128178 2024-11-20T21:28:48,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:28:48,244 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6ef254360cddf7900141ddb056b8fc2e, NAME => 'testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:28:48,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741846_1022 (size=64) 2024-11-20T21:28:48,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741846_1022 (size=64) 2024-11-20T21:28:48,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741846_1022 (size=64) 2024-11-20T21:28:48,674 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:48,677 INFO [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:48,680 INFO [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ef254360cddf7900141ddb056b8fc2e columnFamilyName a 2024-11-20T21:28:48,680 DEBUG [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:48,681 INFO [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] regionserver.HStore(327): Store=6ef254360cddf7900141ddb056b8fc2e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:48,681 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:48,682 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:48,683 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:48,684 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:48,684 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:48,688 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:48,692 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:48,694 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6ef254360cddf7900141ddb056b8fc2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63351221, jitterRate=-0.05599324405193329}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:28:48,695 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6ef254360cddf7900141ddb056b8fc2e: Writing region info on filesystem at 1732138128674Initializing all the Stores at 1732138128676 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138128676Cleaning up temporary data from old regions at 1732138128684 (+8 ms)Region opened successfully at 1732138128695 (+11 ms) 2024-11-20T21:28:48,695 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6ef254360cddf7900141ddb056b8fc2e, disabling compactions & flushes 2024-11-20T21:28:48,695 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e. 2024-11-20T21:28:48,695 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e. 2024-11-20T21:28:48,695 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e. after waiting 0 ms 2024-11-20T21:28:48,695 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e. 2024-11-20T21:28:48,696 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e. 2024-11-20T21:28:48,696 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6ef254360cddf7900141ddb056b8fc2e: Waiting for close lock at 1732138128695Disabling compacts and flushes for region at 1732138128695Disabling writes for close at 1732138128695Writing region close event to WAL at 1732138128695Closed at 1732138128696 (+1 ms) 2024-11-20T21:28:48,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741845_1021 (size=175) 2024-11-20T21:28:48,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741845_1021 (size=175) 2024-11-20T21:28:48,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741845_1021 (size=175) 2024-11-20T21:28:48,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:28:48,715 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85291737:(num 1732138128178) 2024-11-20T21:28:48,716 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T21:28:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741847_1023 (size=469) 2024-11-20T21:28:48,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741847_1023 (size=469) 2024-11-20T21:28:48,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741847_1023 (size=469) 2024-11-20T21:28:48,766 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-20T21:28:48,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741848_1024 (size=322) 2024-11-20T21:28:48,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741848_1024 (size=322) 2024-11-20T21:28:48,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741848_1024 (size=322) 2024-11-20T21:28:48,852 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2, size=322 (322bytes) 2024-11-20T21:28:48,852 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2 2024-11-20T21:28:48,854 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2 after 2ms 2024-11-20T21:28:48,889 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:48,889 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2 took 37ms 2024-11-20T21:28:48,895 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2 so closing down 2024-11-20T21:28:48,895 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:48,898 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-20T21:28:48,926 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002-wal-2.temp 2024-11-20T21:28:48,926 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:48,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741849_1025 (size=322) 2024-11-20T21:28:48,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741849_1025 (size=322) 2024-11-20T21:28:48,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741849_1025 (size=322) 2024-11-20T21:28:48,940 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T21:28:48,943 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002 2024-11-20T21:28:48,943 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-20T21:28:48,943 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2, journal: Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2, size=322 (322bytes) at 1732138128852Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2 so closing down at 1732138128895 (+43 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002-wal-2.temp at 1732138128926 (+31 ms)3 split writer threads finished at 1732138128926Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732138128940 (+14 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002 at 1732138128943 (+3 ms)Processed 1 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732138128943 2024-11-20T21:28:48,965 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1, size=469 (469bytes) 2024-11-20T21:28:48,965 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1 2024-11-20T21:28:48,966 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1 after 1ms 2024-11-20T21:28:48,997 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:48,997 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1 took 32ms 2024-11-20T21:28:49,000 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1 so closing down 2024-11-20T21:28:49,000 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:49,002 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-20T21:28:49,038 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000001-wal-1.temp 2024-11-20T21:28:49,038 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:49,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741850_1026 (size=469) 2024-11-20T21:28:49,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741850_1026 (size=469) 2024-11-20T21:28:49,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741850_1026 (size=469) 2024-11-20T21:28:49,065 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T21:28:49,107 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:49,110 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002, length=322 2024-11-20T21:28:49,113 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002 2024-11-20T21:28:49,113 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-20T21:28:49,114 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1, journal: Splitting hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1, size=469 (469bytes) at 1732138128965Finishing writing output for hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1 so closing down at 1732138129000 (+35 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000001-wal-1.temp at 1732138129038 (+38 ms)3 split writer threads finished at 1732138129039 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732138129065 (+26 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002 at 1732138129113 (+48 ms)Processed 2 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732138129113 2024-11-20T21:28:49,114 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:49,117 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:49,138 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal.1732138129118, exclude list is [], retry=0 2024-11-20T21:28:49,144 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:49,145 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:49,145 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:49,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal.1732138129118 2024-11-20T21:28:49,199 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729)] 2024-11-20T21:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ef254360cddf7900141ddb056b8fc2e, NAME => 'testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,200 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,206 INFO [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,209 INFO [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ef254360cddf7900141ddb056b8fc2e columnFamilyName a 2024-11-20T21:28:49,209 DEBUG [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:49,210 INFO [StoreOpener-6ef254360cddf7900141ddb056b8fc2e-1 {}] regionserver.HStore(327): Store=6ef254360cddf7900141ddb056b8fc2e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:49,210 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,216 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002 2024-11-20T21:28:49,261 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:49,267 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002 2024-11-20T21:28:49,267 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6ef254360cddf7900141ddb056b8fc2e 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-20T21:28:49,291 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/.tmp/a/7ee49a6718c943f48ca06d4792b50aa9 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732138128715/Put/seqid=0 2024-11-20T21:28:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741852_1028 (size=5170) 2024-11-20T21:28:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741852_1028 (size=5170) 2024-11-20T21:28:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741852_1028 (size=5170) 2024-11-20T21:28:49,326 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/.tmp/a/7ee49a6718c943f48ca06d4792b50aa9 2024-11-20T21:28:49,337 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/.tmp/a/7ee49a6718c943f48ca06d4792b50aa9 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/a/7ee49a6718c943f48ca06d4792b50aa9 2024-11-20T21:28:49,348 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/a/7ee49a6718c943f48ca06d4792b50aa9, entries=2, sequenceid=2, filesize=5.0 K 2024-11-20T21:28:49,348 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6ef254360cddf7900141ddb056b8fc2e in 81ms, sequenceid=2, compaction requested=false; wal=null 2024-11-20T21:28:49,349 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/0000000000000000002 2024-11-20T21:28:49,350 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,350 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,352 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6ef254360cddf7900141ddb056b8fc2e 2024-11-20T21:28:49,355 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/6ef254360cddf7900141ddb056b8fc2e/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-20T21:28:49,356 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6ef254360cddf7900141ddb056b8fc2e; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73850789, jitterRate=0.10046251118183136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:28:49,356 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6ef254360cddf7900141ddb056b8fc2e: Writing region info on filesystem at 1732138129200Initializing all the Stores at 1732138129206 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138129206Obtaining lock to block concurrent updates at 1732138129267 (+61 ms)Preparing flush snapshotting stores in 6ef254360cddf7900141ddb056b8fc2e at 1732138129267Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732138129267Flushing stores of testReplayEditsWrittenIntoWAL,,1732138128155.6ef254360cddf7900141ddb056b8fc2e. at 1732138129267Flushing 6ef254360cddf7900141ddb056b8fc2e/a: creating writer at 1732138129267Flushing 6ef254360cddf7900141ddb056b8fc2e/a: appending metadata at 1732138129288 (+21 ms)Flushing 6ef254360cddf7900141ddb056b8fc2e/a: closing flushed file at 1732138129288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c79b65a: reopening flushed file at 1732138129336 (+48 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6ef254360cddf7900141ddb056b8fc2e in 81ms, sequenceid=2, compaction requested=false; wal=null at 1732138129348 (+12 ms)Cleaning up temporary data from old regions at 1732138129350 (+2 ms)Region opened successfully at 1732138129356 (+6 ms) 2024-11-20T21:28:49,385 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:39976 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:52698 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57068 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57178 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:32838 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=641 (was 641), ProcessCount=11 (was 11), AvailableMemoryMB=4549 (was 3531) - AvailableMemoryMB LEAK? - 2024-11-20T21:28:49,402 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=641, ProcessCount=11, AvailableMemoryMB=4543 2024-11-20T21:28:49,424 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:49,429 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:49,430 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:49,435 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29649302, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-29649302, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:49,455 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29649302/hregion-29649302.1732138129436, exclude list is [], retry=0 2024-11-20T21:28:49,460 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:49,461 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:49,468 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:49,517 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29649302/hregion-29649302.1732138129436 2024-11-20T21:28:49,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:49,521 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => efe6bf84b2372ed2ca6f2269d0679cec, NAME => 'testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:28:49,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741854_1030 (size=64) 2024-11-20T21:28:49,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741854_1030 (size=64) 2024-11-20T21:28:49,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741854_1030 (size=64) 2024-11-20T21:28:49,538 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:49,540 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,542 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efe6bf84b2372ed2ca6f2269d0679cec columnFamilyName a 2024-11-20T21:28:49,542 DEBUG [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:49,543 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(327): Store=efe6bf84b2372ed2ca6f2269d0679cec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:49,543 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,545 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efe6bf84b2372ed2ca6f2269d0679cec columnFamilyName b 2024-11-20T21:28:49,545 DEBUG [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:49,545 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(327): Store=efe6bf84b2372ed2ca6f2269d0679cec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:49,546 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,547 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efe6bf84b2372ed2ca6f2269d0679cec columnFamilyName c 2024-11-20T21:28:49,547 DEBUG [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:49,548 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(327): Store=efe6bf84b2372ed2ca6f2269d0679cec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:49,548 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,549 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,549 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,551 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,551 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,552 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:28:49,553 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:49,557 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:49,558 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened efe6bf84b2372ed2ca6f2269d0679cec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60493670, jitterRate=-0.09857407212257385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:28:49,558 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for efe6bf84b2372ed2ca6f2269d0679cec: Writing region info on filesystem at 1732138129539Initializing all the Stores at 1732138129540 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138129540Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138129540Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138129540Cleaning up temporary data from old regions at 1732138129551 (+11 ms)Region opened successfully at 1732138129558 (+7 ms) 2024-11-20T21:28:49,558 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing efe6bf84b2372ed2ca6f2269d0679cec, disabling compactions & flushes 2024-11-20T21:28:49,558 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:49,558 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:49,558 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. after waiting 0 ms 2024-11-20T21:28:49,558 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:49,559 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:49,559 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for efe6bf84b2372ed2ca6f2269d0679cec: Waiting for close lock at 1732138129558Disabling compacts and flushes for region at 1732138129558Disabling writes for close at 1732138129558Writing region close event to WAL at 1732138129559 (+1 ms)Closed at 1732138129559 2024-11-20T21:28:49,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741853_1029 (size=175) 2024-11-20T21:28:49,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741853_1029 (size=175) 2024-11-20T21:28:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741853_1029 (size=175) 2024-11-20T21:28:49,572 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:28:49,572 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29649302:(num 1732138129436) 2024-11-20T21:28:49,573 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:49,575 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:49,596 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576, exclude list is [], retry=0 2024-11-20T21:28:49,601 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:49,601 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:49,602 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:49,649 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 2024-11-20T21:28:49,649 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729)] 2024-11-20T21:28:49,955 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576, size=0 (0bytes) 2024-11-20T21:28:49,956 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 might be still open, length is 0 2024-11-20T21:28:49,956 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 2024-11-20T21:28:49,957 WARN [IPC Server handler 2 on default port 44823 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-20T21:28:49,958 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 after 2ms 2024-11-20T21:28:52,012 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:28:52,067 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T21:28:52,070 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T21:28:52,070 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T21:28:52,072 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:28:52,072 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T21:28:52,073 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T21:28:52,073 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T21:28:52,074 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-20T21:28:52,074 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-20T21:28:52,562 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:52728 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52728 dst: /127.0.0.1:38023 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38023 remote=/127.0.0.1:52728]. Total timeout mills is 60000, 57345 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:28:52,563 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57218 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37559:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57218 dst: /127.0.0.1:37559 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:28:52,563 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:32876 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46339:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32876 dst: /127.0.0.1:46339 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:28:52,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741855_1032 (size=497775) 2024-11-20T21:28:52,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741855_1032 (size=497775) 2024-11-20T21:28:52,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741855_1032 (size=497775) 2024-11-20T21:28:53,959 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 after 4003ms 2024-11-20T21:28:53,996 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:53,998 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 took 4043ms 2024-11-20T21:28:54,013 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732138129576.temp 2024-11-20T21:28:54,055 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000000001-wal.1732138129576.temp 2024-11-20T21:28:54,111 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576; continuing. 2024-11-20T21:28:54,111 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 so closing down 2024-11-20T21:28:54,111 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:54,112 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:54,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741856_1033 (size=497783) 2024-11-20T21:28:54,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741856_1033 (size=497783) 2024-11-20T21:28:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741856_1033 (size=497783) 2024-11-20T21:28:54,130 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000000001-wal.1732138129576.temp (wrote 3002 edits, skipped 0 edits in 41 ms) 2024-11-20T21:28:54,137 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000000001-wal.1732138129576.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000003002 2024-11-20T21:28:54,137 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 138 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T21:28:54,137 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576, journal: Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576, size=0 (0bytes) at 1732138129955Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000000001-wal.1732138129576.temp at 1732138134055 (+4100 ms)Split 1024 edits, skipped 0 edits. at 1732138134061 (+6 ms)Split 2048 edits, skipped 0 edits. at 1732138134086 (+25 ms)Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 so closing down at 1732138134111 (+25 ms)3 split writer threads finished at 1732138134112 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000000001-wal.1732138129576.temp (wrote 3002 edits, skipped 0 edits in 41 ms) at 1732138134130 (+18 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000000001-wal.1732138129576.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000003002 at 1732138134137 (+7 ms)Processed 3002 edits across 1 Regions in 138 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576, size=0, length=0, corrupted=false, cancelled=false at 1732138134137 2024-11-20T21:28:54,154 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138129576 2024-11-20T21:28:54,156 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000003002 2024-11-20T21:28:54,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:54,162 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:54,177 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138134163, exclude list is [], retry=0 2024-11-20T21:28:54,181 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:54,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:54,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:54,225 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138134163 2024-11-20T21:28:54,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:54,226 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:54,228 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:54,230 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efe6bf84b2372ed2ca6f2269d0679cec columnFamilyName a 2024-11-20T21:28:54,230 DEBUG [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:54,231 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(327): Store=efe6bf84b2372ed2ca6f2269d0679cec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:54,231 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:54,232 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efe6bf84b2372ed2ca6f2269d0679cec columnFamilyName b 2024-11-20T21:28:54,232 DEBUG [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:54,233 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(327): Store=efe6bf84b2372ed2ca6f2269d0679cec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:54,233 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:54,234 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efe6bf84b2372ed2ca6f2269d0679cec columnFamilyName c 2024-11-20T21:28:54,234 DEBUG [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:54,235 INFO [StoreOpener-efe6bf84b2372ed2ca6f2269d0679cec-1 {}] regionserver.HStore(327): Store=efe6bf84b2372ed2ca6f2269d0679cec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:54,235 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:54,236 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:54,239 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:54,240 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000003002 2024-11-20T21:28:54,285 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:54,327 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T21:28:54,802 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing efe6bf84b2372ed2ca6f2269d0679cec 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T21:28:54,857 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/a/8f6156a3cacc4fd4a8777d3965854b42 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732138129660/Put/seqid=0 2024-11-20T21:28:54,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741858_1035 (size=50463) 2024-11-20T21:28:54,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741858_1035 (size=50463) 2024-11-20T21:28:54,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741858_1035 (size=50463) 2024-11-20T21:28:54,873 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/a/8f6156a3cacc4fd4a8777d3965854b42 2024-11-20T21:28:54,882 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/a/8f6156a3cacc4fd4a8777d3965854b42 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/a/8f6156a3cacc4fd4a8777d3965854b42 2024-11-20T21:28:54,891 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/a/8f6156a3cacc4fd4a8777d3965854b42, entries=754, sequenceid=754, filesize=49.3 K 2024-11-20T21:28:54,892 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for efe6bf84b2372ed2ca6f2269d0679cec in 89ms, sequenceid=754, compaction requested=false; wal=null 2024-11-20T21:28:54,912 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T21:28:54,912 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing efe6bf84b2372ed2ca6f2269d0679cec 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T21:28:54,923 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/a/d18ce235d0e34530a58b60e1b617918c is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732138129722/Put/seqid=0 2024-11-20T21:28:54,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741859_1036 (size=20072) 2024-11-20T21:28:54,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741859_1036 (size=20072) 2024-11-20T21:28:54,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741859_1036 (size=20072) 2024-11-20T21:28:54,935 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/a/d18ce235d0e34530a58b60e1b617918c 2024-11-20T21:28:54,977 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/b/cc03af7140824ccfb9d2311016c8b93c is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732138129769/Put/seqid=0 2024-11-20T21:28:54,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741860_1037 (size=35835) 2024-11-20T21:28:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741860_1037 (size=35835) 2024-11-20T21:28:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741860_1037 (size=35835) 2024-11-20T21:28:54,994 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/b/cc03af7140824ccfb9d2311016c8b93c 2024-11-20T21:28:55,004 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/a/d18ce235d0e34530a58b60e1b617918c as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/a/d18ce235d0e34530a58b60e1b617918c 2024-11-20T21:28:55,022 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/a/d18ce235d0e34530a58b60e1b617918c, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-20T21:28:55,025 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/b/cc03af7140824ccfb9d2311016c8b93c as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/b/cc03af7140824ccfb9d2311016c8b93c 2024-11-20T21:28:55,037 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/b/cc03af7140824ccfb9d2311016c8b93c, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-20T21:28:55,037 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for efe6bf84b2372ed2ca6f2269d0679cec in 125ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-20T21:28:55,054 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T21:28:55,054 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing efe6bf84b2372ed2ca6f2269d0679cec 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-20T21:28:55,072 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/b/824dc1891f254e5a968a92dadec85c79 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732138129793/Put/seqid=0 2024-11-20T21:28:55,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741861_1038 (size=35082) 2024-11-20T21:28:55,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741861_1038 (size=35082) 2024-11-20T21:28:55,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741861_1038 (size=35082) 2024-11-20T21:28:55,090 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/b/824dc1891f254e5a968a92dadec85c79 2024-11-20T21:28:55,127 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/c/a4aee770fe734f78bf46d2dc1f13b221 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732138129844/Put/seqid=0 2024-11-20T21:28:55,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741862_1039 (size=20825) 2024-11-20T21:28:55,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741862_1039 (size=20825) 2024-11-20T21:28:55,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741862_1039 (size=20825) 2024-11-20T21:28:55,169 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/c/a4aee770fe734f78bf46d2dc1f13b221 2024-11-20T21:28:55,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/b/824dc1891f254e5a968a92dadec85c79 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/b/824dc1891f254e5a968a92dadec85c79 2024-11-20T21:28:55,191 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/b/824dc1891f254e5a968a92dadec85c79, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-20T21:28:55,193 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/c/a4aee770fe734f78bf46d2dc1f13b221 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/c/a4aee770fe734f78bf46d2dc1f13b221 2024-11-20T21:28:55,200 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/c/a4aee770fe734f78bf46d2dc1f13b221, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-20T21:28:55,201 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for efe6bf84b2372ed2ca6f2269d0679cec in 147ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-20T21:28:55,215 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732138129905/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:55,219 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000003002 2024-11-20T21:28:55,220 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-20T21:28:55,220 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing efe6bf84b2372ed2ca6f2269d0679cec 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-20T21:28:55,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/c/12276bbf102d401aa36108e0c86587ad is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732138129853/Put/seqid=0 2024-11-20T21:28:55,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741863_1040 (size=50301) 2024-11-20T21:28:55,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741863_1040 (size=50301) 2024-11-20T21:28:55,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741863_1040 (size=50301) 2024-11-20T21:28:55,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/c/12276bbf102d401aa36108e0c86587ad 2024-11-20T21:28:55,279 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12276bbf102d401aa36108e0c86587ad 2024-11-20T21:28:55,281 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/.tmp/c/12276bbf102d401aa36108e0c86587ad as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/c/12276bbf102d401aa36108e0c86587ad 2024-11-20T21:28:55,292 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 12276bbf102d401aa36108e0c86587ad 2024-11-20T21:28:55,293 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/c/12276bbf102d401aa36108e0c86587ad, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-20T21:28:55,293 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for efe6bf84b2372ed2ca6f2269d0679cec in 73ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-20T21:28:55,294 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/0000000000000003002 2024-11-20T21:28:55,296 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:55,296 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:55,297 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:28:55,300 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for efe6bf84b2372ed2ca6f2269d0679cec 2024-11-20T21:28:55,313 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenIntoWAL/efe6bf84b2372ed2ca6f2269d0679cec/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-20T21:28:55,315 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened efe6bf84b2372ed2ca6f2269d0679cec; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60980925, jitterRate=-0.09131340682506561}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:28:55,315 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for efe6bf84b2372ed2ca6f2269d0679cec: Writing region info on filesystem at 1732138134226Initializing all the Stores at 1732138134228 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138134228Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138134228Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138134228Cleaning up temporary data from old regions at 1732138135296 (+1068 ms)Region opened successfully at 1732138135315 (+19 ms) 2024-11-20T21:28:55,388 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing efe6bf84b2372ed2ca6f2269d0679cec, disabling compactions & flushes 2024-11-20T21:28:55,388 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:55,388 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:55,388 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. after waiting 0 ms 2024-11-20T21:28:55,388 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:55,407 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732138129425.efe6bf84b2372ed2ca6f2269d0679cec. 2024-11-20T21:28:55,407 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for efe6bf84b2372ed2ca6f2269d0679cec: Waiting for close lock at 1732138135388Disabling compacts and flushes for region at 1732138135388Disabling writes for close at 1732138135388Writing region close event to WAL at 1732138135407 (+19 ms)Closed at 1732138135407 2024-11-20T21:28:55,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741857_1034 (size=175) 2024-11-20T21:28:55,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741857_1034 (size=175) 2024-11-20T21:28:55,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741857_1034 (size=175) 2024-11-20T21:28:55,450 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:28:55,450 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732138134163) 2024-11-20T21:28:55,484 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 385) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42443 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1578958332_22 at /127.0.0.1:32960 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34105 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1654203666) connection to localhost/127.0.0.1:44823 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44823 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1654203666) connection to localhost/127.0.0.1:34105 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1578958332_22 at /127.0.0.1:52756 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1578958332_22 at /127.0.0.1:32930 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1578958332_22 at /127.0.0.1:57232 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1654203666) connection to localhost/127.0.0.1:42443 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=629 (was 641), ProcessCount=11 (was 11), AvailableMemoryMB=4265 (was 4543) 2024-11-20T21:28:55,516 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=629, ProcessCount=11, AvailableMemoryMB=4261 2024-11-20T21:28:55,546 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:55,552 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:55,557 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:55,563 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32267759, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-32267759, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:55,585 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32267759/hregion-32267759.1732138135564, exclude list is [], retry=0 2024-11-20T21:28:55,592 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:55,592 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:55,597 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:55,633 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32267759/hregion-32267759.1732138135564 2024-11-20T21:28:55,636 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:55,636 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7736a8cdd60094e876c2ce3475def460, NAME => 'test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:28:55,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741865_1042 (size=43) 2024-11-20T21:28:55,675 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741865_1042 (size=43) 2024-11-20T21:28:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741865_1042 (size=43) 2024-11-20T21:28:55,677 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,679 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7736a8cdd60094e876c2ce3475def460 columnFamilyName a 2024-11-20T21:28:55,679 DEBUG [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:55,679 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(327): Store=7736a8cdd60094e876c2ce3475def460/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:55,680 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,681 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7736a8cdd60094e876c2ce3475def460 columnFamilyName b 2024-11-20T21:28:55,682 DEBUG [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:55,682 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(327): Store=7736a8cdd60094e876c2ce3475def460/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:55,682 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,684 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7736a8cdd60094e876c2ce3475def460 columnFamilyName c 2024-11-20T21:28:55,684 DEBUG [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:55,685 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(327): Store=7736a8cdd60094e876c2ce3475def460/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:55,685 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,686 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,686 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,688 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,688 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,689 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:28:55,690 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:55,693 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:55,693 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7736a8cdd60094e876c2ce3475def460; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68444939, jitterRate=0.019909068942070007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:28:55,695 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7736a8cdd60094e876c2ce3475def460: Writing region info on filesystem at 1732138135675Initializing all the Stores at 1732138135676 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138135676Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138135677 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138135677Cleaning up temporary data from old regions at 1732138135688 (+11 ms)Region opened successfully at 1732138135695 (+7 ms) 2024-11-20T21:28:55,695 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7736a8cdd60094e876c2ce3475def460, disabling compactions & flushes 2024-11-20T21:28:55,695 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:55,695 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:55,695 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. after waiting 0 ms 2024-11-20T21:28:55,695 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:55,696 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:55,696 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7736a8cdd60094e876c2ce3475def460: Waiting for close lock at 1732138135695Disabling compacts and flushes for region at 1732138135695Disabling writes for close at 1732138135695Writing region close event to WAL at 1732138135696 (+1 ms)Closed at 1732138135696 2024-11-20T21:28:55,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741864_1041 (size=175) 2024-11-20T21:28:55,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741864_1041 (size=175) 2024-11-20T21:28:55,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741864_1041 (size=175) 2024-11-20T21:28:55,704 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:28:55,704 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32267759:(num 1732138135564) 2024-11-20T21:28:55,704 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:55,707 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:55,725 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708, exclude list is [], retry=0 2024-11-20T21:28:55,729 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:55,729 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:55,730 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:55,762 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708 2024-11-20T21:28:55,762 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:55,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741866_1043 (size=371388) 2024-11-20T21:28:55,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741866_1043 (size=371388) 2024-11-20T21:28:55,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741866_1043 (size=371388) 2024-11-20T21:28:56,024 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708, size=362.7 K (371388bytes) 2024-11-20T21:28:56,024 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708 2024-11-20T21:28:56,027 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708 after 2ms 2024-11-20T21:28:56,104 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:56,107 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708 took 82ms 2024-11-20T21:28:56,145 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732138135708.temp 2024-11-20T21:28:56,182 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708 so closing down 2024-11-20T21:28:56,182 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:56,206 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000000001-wal.1732138135708.temp 2024-11-20T21:28:56,225 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:56,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741867_1044 (size=371388) 2024-11-20T21:28:56,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741867_1044 (size=371388) 2024-11-20T21:28:56,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741867_1044 (size=371388) 2024-11-20T21:28:56,231 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000000001-wal.1732138135708.temp (wrote 3000 edits, skipped 0 edits in 18 ms) 2024-11-20T21:28:56,233 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000000001-wal.1732138135708.temp to hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003000 2024-11-20T21:28:56,233 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 104 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-20T21:28:56,233 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708, journal: Splitting hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708, size=362.7 K (371388bytes) at 1732138136024Split 1024 edits, skipped 0 edits. at 1732138136151 (+127 ms)Split 2048 edits, skipped 0 edits. at 1732138136168 (+17 ms)Finishing writing output for hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708 so closing down at 1732138136182 (+14 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000000001-wal.1732138135708.temp at 1732138136206 (+24 ms)3 split writer threads finished at 1732138136225 (+19 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000000001-wal.1732138135708.temp (wrote 3000 edits, skipped 0 edits in 18 ms) at 1732138136231 (+6 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000000001-wal.1732138135708.temp to hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003000 at 1732138136233 (+2 ms)Processed 3000 edits across 1 Regions in 104 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732138136233 2024-11-20T21:28:56,235 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138135708 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138135708 2024-11-20T21:28:56,236 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003000 2024-11-20T21:28:56,237 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:56,239 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:56,254 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240, exclude list is [], retry=0 2024-11-20T21:28:56,258 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:56,258 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:56,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:56,290 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240 2024-11-20T21:28:56,290 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:28:56,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741868_1045 (size=371515) 2024-11-20T21:28:56,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741868_1045 (size=371515) 2024-11-20T21:28:56,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741868_1045 (size=371515) 2024-11-20T21:28:56,493 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240, size=362.8 K (371515bytes) 2024-11-20T21:28:56,493 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240 2024-11-20T21:28:56,493 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240 after 0ms 2024-11-20T21:28:56,537 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:56,538 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240 took 46ms 2024-11-20T21:28:56,547 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732138136240.temp 2024-11-20T21:28:56,602 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240 so closing down 2024-11-20T21:28:56,602 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:56,626 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003001-wal.1732138136240.temp 2024-11-20T21:28:56,637 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:56,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741869_1046 (size=371515) 2024-11-20T21:28:56,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741869_1046 (size=371515) 2024-11-20T21:28:56,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741869_1046 (size=371515) 2024-11-20T21:28:56,656 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003001-wal.1732138136240.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-20T21:28:56,658 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003001-wal.1732138136240.temp to hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000006000 2024-11-20T21:28:56,659 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 120 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-20T21:28:56,659 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240, journal: Splitting hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240, size=362.8 K (371515bytes) at 1732138136493Split 1024 edits, skipped 0 edits. at 1732138136580 (+87 ms)Split 2048 edits, skipped 0 edits. at 1732138136590 (+10 ms)Finishing writing output for hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240 so closing down at 1732138136602 (+12 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003001-wal.1732138136240.temp at 1732138136626 (+24 ms)3 split writer threads finished at 1732138136637 (+11 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003001-wal.1732138136240.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1732138136656 (+19 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003001-wal.1732138136240.temp to hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000006000 at 1732138136659 (+3 ms)Processed 3000 edits across 1 Regions in 120 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732138136659 2024-11-20T21:28:56,663 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136240 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138136240 2024-11-20T21:28:56,666 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000006000 2024-11-20T21:28:56,666 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:56,670 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/test2727-manual,16010,1732138135544, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:56,690 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136671, exclude list is [], retry=0 2024-11-20T21:28:56,694 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:56,694 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:56,694 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:56,733 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732138135544/wal.1732138136671 2024-11-20T21:28:56,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729)] 2024-11-20T21:28:56,737 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7736a8cdd60094e876c2ce3475def460, NAME => 'test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:28:56,737 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:56,737 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,737 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,741 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,744 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7736a8cdd60094e876c2ce3475def460 columnFamilyName a 2024-11-20T21:28:56,748 DEBUG [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:56,749 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(327): Store=7736a8cdd60094e876c2ce3475def460/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:56,749 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,751 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7736a8cdd60094e876c2ce3475def460 columnFamilyName b 2024-11-20T21:28:56,751 DEBUG [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:56,752 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(327): Store=7736a8cdd60094e876c2ce3475def460/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:56,752 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,753 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7736a8cdd60094e876c2ce3475def460 columnFamilyName c 2024-11-20T21:28:56,753 DEBUG [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:56,754 INFO [StoreOpener-7736a8cdd60094e876c2ce3475def460-1 {}] regionserver.HStore(327): Store=7736a8cdd60094e876c2ce3475def460/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:56,754 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:56,771 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003000 2024-11-20T21:28:56,808 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:56,847 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003000 2024-11-20T21:28:56,848 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000006000 2024-11-20T21:28:56,884 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:56,920 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000006000 2024-11-20T21:28:56,921 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7736a8cdd60094e876c2ce3475def460 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-20T21:28:56,951 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/a/0493a0b8d8414ef485aa8c27733f4ee8 is 41, key is test2727/a:100/1732138136296/Put/seqid=0 2024-11-20T21:28:57,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741871_1048 (size=84227) 2024-11-20T21:28:57,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741871_1048 (size=84227) 2024-11-20T21:28:57,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741871_1048 (size=84227) 2024-11-20T21:28:57,027 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/a/0493a0b8d8414ef485aa8c27733f4ee8 2024-11-20T21:28:57,076 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/b/ae535c8de71d4031ba0afda5c095cc1a is 41, key is test2727/b:100/1732138136346/Put/seqid=0 2024-11-20T21:28:57,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741872_1049 (size=84609) 2024-11-20T21:28:57,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741872_1049 (size=84609) 2024-11-20T21:28:57,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741872_1049 (size=84609) 2024-11-20T21:28:57,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/b/ae535c8de71d4031ba0afda5c095cc1a 2024-11-20T21:28:57,598 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/c/290bc2ad0a234080bc4fc92d725d47a6 is 41, key is test2727/c:100/1732138136405/Put/seqid=0 2024-11-20T21:28:57,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741873_1050 (size=84609) 2024-11-20T21:28:57,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741873_1050 (size=84609) 2024-11-20T21:28:57,635 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/c/290bc2ad0a234080bc4fc92d725d47a6 2024-11-20T21:28:57,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741873_1050 (size=84609) 2024-11-20T21:28:57,644 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/a/0493a0b8d8414ef485aa8c27733f4ee8 as hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/a/0493a0b8d8414ef485aa8c27733f4ee8 2024-11-20T21:28:57,656 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/a/0493a0b8d8414ef485aa8c27733f4ee8, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-20T21:28:57,658 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/b/ae535c8de71d4031ba0afda5c095cc1a as hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/b/ae535c8de71d4031ba0afda5c095cc1a 2024-11-20T21:28:57,680 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/b/ae535c8de71d4031ba0afda5c095cc1a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T21:28:57,682 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/.tmp/c/290bc2ad0a234080bc4fc92d725d47a6 as hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/c/290bc2ad0a234080bc4fc92d725d47a6 2024-11-20T21:28:57,703 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/c/290bc2ad0a234080bc4fc92d725d47a6, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-20T21:28:57,704 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 7736a8cdd60094e876c2ce3475def460 in 783ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-20T21:28:57,705 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000003000 2024-11-20T21:28:57,706 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/0000000000000006000 2024-11-20T21:28:57,708 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:57,708 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:57,710 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:28:57,714 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7736a8cdd60094e876c2ce3475def460 2024-11-20T21:28:57,725 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/test2727/7736a8cdd60094e876c2ce3475def460/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-20T21:28:57,726 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7736a8cdd60094e876c2ce3475def460; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73039692, jitterRate=0.08837622404098511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:28:57,727 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7736a8cdd60094e876c2ce3475def460: Writing region info on filesystem at 1732138136737Initializing all the Stores at 1732138136741 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138136741Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138136741Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138136741Obtaining lock to block concurrent updates at 1732138136921 (+180 ms)Preparing flush snapshotting stores in 7736a8cdd60094e876c2ce3475def460 at 1732138136921Finished memstore snapshotting test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732138136921Flushing stores of test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. at 1732138136921Flushing 7736a8cdd60094e876c2ce3475def460/a: creating writer at 1732138136922 (+1 ms)Flushing 7736a8cdd60094e876c2ce3475def460/a: appending metadata at 1732138136950 (+28 ms)Flushing 7736a8cdd60094e876c2ce3475def460/a: closing flushed file at 1732138136950Flushing 7736a8cdd60094e876c2ce3475def460/b: creating writer at 1732138137047 (+97 ms)Flushing 7736a8cdd60094e876c2ce3475def460/b: appending metadata at 1732138137073 (+26 ms)Flushing 7736a8cdd60094e876c2ce3475def460/b: closing flushed file at 1732138137073Flushing 7736a8cdd60094e876c2ce3475def460/c: creating writer at 1732138137569 (+496 ms)Flushing 7736a8cdd60094e876c2ce3475def460/c: appending metadata at 1732138137597 (+28 ms)Flushing 7736a8cdd60094e876c2ce3475def460/c: closing flushed file at 1732138137597Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72530b3b: reopening flushed file at 1732138137643 (+46 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ddc83f3: reopening flushed file at 1732138137656 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43e869e: reopening flushed file at 1732138137680 (+24 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 7736a8cdd60094e876c2ce3475def460 in 783ms, sequenceid=6000, compaction requested=false; wal=null at 1732138137704 (+24 ms)Cleaning up temporary data from old regions at 1732138137708 (+4 ms)Region opened successfully at 1732138137727 (+19 ms) 2024-11-20T21:28:57,729 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-20T21:28:57,729 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7736a8cdd60094e876c2ce3475def460, disabling compactions & flushes 2024-11-20T21:28:57,729 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:57,729 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:57,729 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. after waiting 0 ms 2024-11-20T21:28:57,729 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:57,742 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732138135549.7736a8cdd60094e876c2ce3475def460. 2024-11-20T21:28:57,742 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7736a8cdd60094e876c2ce3475def460: Waiting for close lock at 1732138137729Disabling compacts and flushes for region at 1732138137729Disabling writes for close at 1732138137729Writing region close event to WAL at 1732138137742 (+13 ms)Closed at 1732138137742 2024-11-20T21:28:57,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741870_1047 (size=175) 2024-11-20T21:28:57,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741870_1047 (size=175) 2024-11-20T21:28:57,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741870_1047 (size=175) 2024-11-20T21:28:57,751 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:28:57,751 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732138136671) 2024-11-20T21:28:57,770 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=397 (was 399), OpenFileDescriptor=921 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=619 (was 629), ProcessCount=11 (was 11), AvailableMemoryMB=3893 (was 4261) 2024-11-20T21:28:57,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=397, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=619, ProcessCount=11, AvailableMemoryMB=3893 2024-11-20T21:28:57,808 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:57,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:57,817 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732138137817 2024-11-20T21:28:57,868 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817 2024-11-20T21:28:57,877 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:57,883 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b30c2809aa8d86d215be3ec9bc0dea5, NAME => 'testSequentialEditLogSeqNum,,1732138137809.9b30c2809aa8d86d215be3ec9bc0dea5.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:28:57,883 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732138137809.9b30c2809aa8d86d215be3ec9bc0dea5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:57,883 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,883 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,884 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5 doesn't exist for region: 9b30c2809aa8d86d215be3ec9bc0dea5 on table testSequentialEditLogSeqNum 2024-11-20T21:28:57,886 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 9b30c2809aa8d86d215be3ec9bc0dea5 on table testSequentialEditLogSeqNum 2024-11-20T21:28:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741875_1052 (size=62) 2024-11-20T21:28:57,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741875_1052 (size=62) 2024-11-20T21:28:57,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741875_1052 (size=62) 2024-11-20T21:28:57,912 INFO [StoreOpener-9b30c2809aa8d86d215be3ec9bc0dea5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,918 INFO [StoreOpener-9b30c2809aa8d86d215be3ec9bc0dea5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b30c2809aa8d86d215be3ec9bc0dea5 columnFamilyName a 2024-11-20T21:28:57,918 DEBUG [StoreOpener-9b30c2809aa8d86d215be3ec9bc0dea5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:57,918 INFO [StoreOpener-9b30c2809aa8d86d215be3ec9bc0dea5-1 {}] regionserver.HStore(327): Store=9b30c2809aa8d86d215be3ec9bc0dea5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:57,919 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,921 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,927 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,929 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,929 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,934 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b30c2809aa8d86d215be3ec9bc0dea5 2024-11-20T21:28:57,957 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:57,958 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b30c2809aa8d86d215be3ec9bc0dea5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64136983, jitterRate=-0.044284477829933167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:28:57,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b30c2809aa8d86d215be3ec9bc0dea5: Writing region info on filesystem at 1732138137883Initializing all the Stores at 1732138137911 (+28 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138137911Cleaning up temporary data from old regions at 1732138137929 (+18 ms)Region opened successfully at 1732138137960 (+31 ms) 2024-11-20T21:28:57,988 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9b30c2809aa8d86d215be3ec9bc0dea5 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-20T21:28:58,026 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/.tmp/a/1274d3848f9249c2b03577f934bbb93d is 81, key is testSequentialEditLogSeqNum/a:x0/1732138137960/Put/seqid=0 2024-11-20T21:28:58,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741876_1053 (size=5833) 2024-11-20T21:28:58,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741876_1053 (size=5833) 2024-11-20T21:28:58,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741876_1053 (size=5833) 2024-11-20T21:28:58,036 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/.tmp/a/1274d3848f9249c2b03577f934bbb93d 2024-11-20T21:28:58,043 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/.tmp/a/1274d3848f9249c2b03577f934bbb93d as hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/a/1274d3848f9249c2b03577f934bbb93d 2024-11-20T21:28:58,050 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/a/1274d3848f9249c2b03577f934bbb93d, entries=10, sequenceid=13, filesize=5.7 K 2024-11-20T21:28:58,051 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9b30c2809aa8d86d215be3ec9bc0dea5 in 63ms, sequenceid=13, compaction requested=false 2024-11-20T21:28:58,051 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9b30c2809aa8d86d215be3ec9bc0dea5: 2024-11-20T21:28:58,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:28:58,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:28:58,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:28:58,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:28:58,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-20T21:28:58,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741874_1051 (size=3256) 2024-11-20T21:28:58,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741874_1051 (size=3256) 2024-11-20T21:28:58,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741874_1051 (size=3256) 2024-11-20T21:28:58,083 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817, size=3.2 K (3256bytes) 2024-11-20T21:28:58,084 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817 2024-11-20T21:28:58,084 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817 after 0ms 2024-11-20T21:28:58,119 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:28:58,120 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817 took 36ms 2024-11-20T21:28:58,122 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817 so closing down 2024-11-20T21:28:58,123 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:28:58,124 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732138137817.temp 2024-11-20T21:28:58,151 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000003-wal.1732138137817.temp 2024-11-20T21:28:58,152 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:28:58,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741877_1054 (size=2650) 2024-11-20T21:28:58,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741877_1054 (size=2650) 2024-11-20T21:28:58,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741877_1054 (size=2650) 2024-11-20T21:28:58,159 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000003-wal.1732138137817.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T21:28:58,161 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000003-wal.1732138137817.temp to hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000020 2024-11-20T21:28:58,161 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-20T21:28:58,161 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817, journal: Splitting hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817, size=3.2 K (3256bytes) at 1732138138083Finishing writing output for hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817 so closing down at 1732138138122 (+39 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000003-wal.1732138137817.temp at 1732138138151 (+29 ms)3 split writer threads finished at 1732138138152 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000003-wal.1732138137817.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732138138159 (+7 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000003-wal.1732138137817.temp to hdfs://localhost:44823/hbase/data/default/testSequentialEditLogSeqNum/9b30c2809aa8d86d215be3ec9bc0dea5/recovered.edits/0000000000000000020 at 1732138138161 (+2 ms)Processed 17 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:44823/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732138137806/wal.1732138137817, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732138138161 2024-11-20T21:28:58,175 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=402 (was 397) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:60414 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57232 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=619 (was 619), ProcessCount=11 (was 11), AvailableMemoryMB=3874 (was 3893) 2024-11-20T21:28:58,188 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=402, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=619, ProcessCount=11, AvailableMemoryMB=3874 2024-11-20T21:28:58,206 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:58,209 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:28:58,237 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:28:58,240 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12354601, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-12354601, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:58,257 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12354601/hregion-12354601.1732138138241, exclude list is [], retry=0 2024-11-20T21:28:58,262 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:58,262 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:58,264 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:58,304 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12354601/hregion-12354601.1732138138241 2024-11-20T21:28:58,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:28:58,304 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 20304ebc3b9c9c044192e4d4665eeeba, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:28:58,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741879_1056 (size=70) 2024-11-20T21:28:58,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741879_1056 (size=70) 2024-11-20T21:28:58,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741879_1056 (size=70) 2024-11-20T21:28:58,315 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:58,317 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,318 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName a 2024-11-20T21:28:58,318 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:58,319 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:58,319 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,320 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName b 2024-11-20T21:28:58,320 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:58,321 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:58,321 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,323 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName c 2024-11-20T21:28:58,323 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:58,323 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:58,324 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,325 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,326 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,326 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,327 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:28:58,328 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,331 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:28:58,332 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20304ebc3b9c9c044192e4d4665eeeba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73986845, jitterRate=0.1024899035692215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:28:58,333 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20304ebc3b9c9c044192e4d4665eeeba: Writing region info on filesystem at 1732138138315Initializing all the Stores at 1732138138316 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138138316Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138138316Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138138316Cleaning up temporary data from old regions at 1732138138326 (+10 ms)Region opened successfully at 1732138138333 (+7 ms) 2024-11-20T21:28:58,333 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 20304ebc3b9c9c044192e4d4665eeeba, disabling compactions & flushes 2024-11-20T21:28:58,334 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:28:58,334 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:28:58,334 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. after waiting 0 ms 2024-11-20T21:28:58,334 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:28:58,334 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:28:58,334 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 20304ebc3b9c9c044192e4d4665eeeba: Waiting for close lock at 1732138138333Disabling compacts and flushes for region at 1732138138333Disabling writes for close at 1732138138334 (+1 ms)Writing region close event to WAL at 1732138138334Closed at 1732138138334 2024-11-20T21:28:58,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741878_1055 (size=175) 2024-11-20T21:28:58,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741878_1055 (size=175) 2024-11-20T21:28:58,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741878_1055 (size=175) 2024-11-20T21:28:58,347 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:28:58,347 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12354601:(num 1732138138241) 2024-11-20T21:28:58,347 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:28:58,350 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:28:58,368 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350, exclude list is [], retry=0 2024-11-20T21:28:58,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:28:58,372 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:28:58,372 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:28:58,401 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 2024-11-20T21:28:58,401 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:28:58,401 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 20304ebc3b9c9c044192e4d4665eeeba, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:28:58,402 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:28:58,402 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,402 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,403 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,405 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName a 2024-11-20T21:28:58,405 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:58,405 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:58,405 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,406 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName b 2024-11-20T21:28:58,407 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:58,407 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:58,407 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,408 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName c 2024-11-20T21:28:58,408 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:28:58,408 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:28:58,409 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,412 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,412 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,412 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:28:58,414 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:28:58,415 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20304ebc3b9c9c044192e4d4665eeeba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61120172, jitterRate=-0.08923846483230591}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:28:58,415 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20304ebc3b9c9c044192e4d4665eeeba: Writing region info on filesystem at 1732138138402Initializing all the Stores at 1732138138403 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138138403Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138138403Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138138403Cleaning up temporary data from old regions at 1732138138412 (+9 ms)Region opened successfully at 1732138138415 (+3 ms) 2024-11-20T21:28:58,419 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732138138418/Put/seqid=0 2024-11-20T21:28:58,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741881_1058 (size=4826) 2024-11-20T21:28:58,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741881_1058 (size=4826) 2024-11-20T21:28:58,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741881_1058 (size=4826) 2024-11-20T21:28:58,431 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44823/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 20304ebc3b9c9c044192e4d4665eeeba/a 2024-11-20T21:28:58,442 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-20T21:28:58,442 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T21:28:58,443 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 20304ebc3b9c9c044192e4d4665eeeba: 2024-11-20T21:28:58,445 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/a/7a8b0bac2e6b4108bf6c000b2ad61dbc_SeqId_3_ 2024-11-20T21:28:58,446 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44823/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 20304ebc3b9c9c044192e4d4665eeeba/a as hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/a/7a8b0bac2e6b4108bf6c000b2ad61dbc_SeqId_3_ - updating store file list. 2024-11-20T21:28:58,455 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7a8b0bac2e6b4108bf6c000b2ad61dbc_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T21:28:58,456 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/a/7a8b0bac2e6b4108bf6c000b2ad61dbc_SeqId_3_ into 20304ebc3b9c9c044192e4d4665eeeba/a 2024-11-20T21:28:58,456 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44823/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 20304ebc3b9c9c044192e4d4665eeeba/a (new location: hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/a/7a8b0bac2e6b4108bf6c000b2ad61dbc_SeqId_3_) 2024-11-20T21:28:58,525 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350, size=0 (0bytes) 2024-11-20T21:28:58,525 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 might be still open, length is 0 2024-11-20T21:28:58,525 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 2024-11-20T21:28:58,526 WARN [IPC Server handler 3 on default port 44823 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-20T21:28:58,526 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 after 1ms 2024-11-20T21:28:58,557 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57684 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46339:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57684 dst: /127.0.0.1:46339 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46339 remote=/127.0.0.1:57684]. Total timeout mills is 60000, 59910 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:28:58,558 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:58736 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37559:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58736 dst: /127.0.0.1:37559 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:28:58,561 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:60498 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60498 dst: /127.0.0.1:38023 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:28:58,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741880_1059 (size=606) 2024-11-20T21:28:58,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741880_1059 (size=606) 2024-11-20T21:29:02,057 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-20T21:29:02,057 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-20T21:29:02,059 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-20T21:29:02,059 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-20T21:29:02,527 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 after 4002ms 2024-11-20T21:29:02,558 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:02,558 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 took 4033ms 2024-11-20T21:29:02,560 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350; continuing. 2024-11-20T21:29:02,560 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 so closing down 2024-11-20T21:29:02,560 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:02,563 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732138138350.temp 2024-11-20T21:29:02,594 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005-wal.1732138138350.temp 2024-11-20T21:29:02,595 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:02,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741882_1060 (size=329) 2024-11-20T21:29:02,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741882_1060 (size=329) 2024-11-20T21:29:02,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741882_1060 (size=329) 2024-11-20T21:29:02,603 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005-wal.1732138138350.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:02,604 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005-wal.1732138138350.temp to hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005 2024-11-20T21:29:02,605 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T21:29:02,605 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350, journal: Splitting hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350, size=0 (0bytes) at 1732138138525Finishing writing output for hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 so closing down at 1732138142560 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005-wal.1732138138350.temp at 1732138142594 (+34 ms)3 split writer threads finished at 1732138142595 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005-wal.1732138138350.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732138142603 (+8 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005-wal.1732138138350.temp to hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005 at 1732138142604 (+1 ms)Processed 2 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350, size=0, length=0, corrupted=false, cancelled=false at 1732138142605 (+1 ms) 2024-11-20T21:29:02,607 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138138350 2024-11-20T21:29:02,608 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005 2024-11-20T21:29:02,608 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:02,610 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:02,625 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138142610, exclude list is [], retry=0 2024-11-20T21:29:02,628 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:02,628 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:02,628 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:02,661 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138142610 2024-11-20T21:29:02,662 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:29:02,662 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 20304ebc3b9c9c044192e4d4665eeeba, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:02,662 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:02,662 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,662 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,664 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,665 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName a 2024-11-20T21:29:02,666 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:02,673 DEBUG [StoreFileOpener-20304ebc3b9c9c044192e4d4665eeeba-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7a8b0bac2e6b4108bf6c000b2ad61dbc_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-20T21:29:02,673 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/a/7a8b0bac2e6b4108bf6c000b2ad61dbc_SeqId_3_ 2024-11-20T21:29:02,673 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:02,673 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,675 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName b 2024-11-20T21:29:02,675 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:02,675 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:02,675 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,677 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20304ebc3b9c9c044192e4d4665eeeba columnFamilyName c 2024-11-20T21:29:02,677 DEBUG [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:02,678 INFO [StoreOpener-20304ebc3b9c9c044192e4d4665eeeba-1 {}] regionserver.HStore(327): Store=20304ebc3b9c9c044192e4d4665eeeba/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:02,678 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,679 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,680 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,681 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005 2024-11-20T21:29:02,711 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:02,712 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005 2024-11-20T21:29:02,712 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 20304ebc3b9c9c044192e4d4665eeeba 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-20T21:29:02,738 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/.tmp/a/16a79cdcf4b24484b32292b9568e09ea is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732138138463/Put/seqid=0 2024-11-20T21:29:02,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741884_1062 (size=5149) 2024-11-20T21:29:02,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741884_1062 (size=5149) 2024-11-20T21:29:02,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741884_1062 (size=5149) 2024-11-20T21:29:02,748 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/.tmp/a/16a79cdcf4b24484b32292b9568e09ea 2024-11-20T21:29:02,763 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/.tmp/a/16a79cdcf4b24484b32292b9568e09ea as hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/a/16a79cdcf4b24484b32292b9568e09ea 2024-11-20T21:29:02,769 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/a/16a79cdcf4b24484b32292b9568e09ea, entries=1, sequenceid=5, filesize=5.0 K 2024-11-20T21:29:02,769 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 20304ebc3b9c9c044192e4d4665eeeba in 57ms, sequenceid=5, compaction requested=false; wal=null 2024-11-20T21:29:02,770 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/0000000000000000005 2024-11-20T21:29:02,771 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,771 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,772 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:02,773 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20304ebc3b9c9c044192e4d4665eeeba 2024-11-20T21:29:02,775 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20304ebc3b9c9c044192e4d4665eeeba/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-20T21:29:02,776 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20304ebc3b9c9c044192e4d4665eeeba; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61271016, jitterRate=-0.08699071407318115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:02,777 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20304ebc3b9c9c044192e4d4665eeeba: Writing region info on filesystem at 1732138142663Initializing all the Stores at 1732138142664 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138142664Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138142664Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138142664Obtaining lock to block concurrent updates at 1732138142712 (+48 ms)Preparing flush snapshotting stores in 20304ebc3b9c9c044192e4d4665eeeba at 1732138142712Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732138142713 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. at 1732138142713Flushing 20304ebc3b9c9c044192e4d4665eeeba/a: creating writer at 1732138142713Flushing 20304ebc3b9c9c044192e4d4665eeeba/a: appending metadata at 1732138142737 (+24 ms)Flushing 20304ebc3b9c9c044192e4d4665eeeba/a: closing flushed file at 1732138142737Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@493ce818: reopening flushed file at 1732138142762 (+25 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 20304ebc3b9c9c044192e4d4665eeeba in 57ms, sequenceid=5, compaction requested=false; wal=null at 1732138142769 (+7 ms)Cleaning up temporary data from old regions at 1732138142771 (+2 ms)Region opened successfully at 1732138142777 (+6 ms) 2024-11-20T21:29:02,781 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 20304ebc3b9c9c044192e4d4665eeeba, disabling compactions & flushes 2024-11-20T21:29:02,781 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:29:02,781 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:29:02,781 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. after waiting 0 ms 2024-11-20T21:29:02,781 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:29:02,782 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732138138207.20304ebc3b9c9c044192e4d4665eeeba. 2024-11-20T21:29:02,783 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 20304ebc3b9c9c044192e4d4665eeeba: Waiting for close lock at 1732138142781Disabling compacts and flushes for region at 1732138142781Disabling writes for close at 1732138142781Writing region close event to WAL at 1732138142782 (+1 ms)Closed at 1732138142782 2024-11-20T21:29:02,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741883_1061 (size=175) 2024-11-20T21:29:02,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741883_1061 (size=175) 2024-11-20T21:29:02,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741883_1061 (size=175) 2024-11-20T21:29:02,788 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:02,788 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732138142610) 2024-11-20T21:29:02,804 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 402) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1615548632_22 at /127.0.0.1:58754 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44823 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1615548632_22 at /127.0.0.1:60554 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1615548632_22 at /127.0.0.1:57728 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1654203666) connection to localhost/127.0.0.1:44823 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=617 (was 619), ProcessCount=11 (was 11), AvailableMemoryMB=3630 (was 3874) 2024-11-20T21:29:02,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=617, ProcessCount=11, AvailableMemoryMB=3627 2024-11-20T21:29:02,834 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:02,840 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:29:02,844 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39535,1732138121558 2024-11-20T21:29:02,847 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17ab2aad 2024-11-20T21:29:02,848 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:29:02,850 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:29:02,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:29:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T21:29:02,866 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:29:02,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-20T21:29:02,869 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:02,871 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:29:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:29:02,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741885_1063 (size=694) 2024-11-20T21:29:02,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741885_1063 (size=694) 2024-11-20T21:29:02,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741885_1063 (size=694) 2024-11-20T21:29:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:29:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:29:03,291 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06f91812474a7664c203be2e75d69d0e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c 2024-11-20T21:29:03,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741886_1064 (size=77) 2024-11-20T21:29:03,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741886_1064 (size=77) 2024-11-20T21:29:03,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741886_1064 (size=77) 2024-11-20T21:29:03,302 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:03,302 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 06f91812474a7664c203be2e75d69d0e, disabling compactions & flushes 2024-11-20T21:29:03,302 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:03,303 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:03,303 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. after waiting 0 ms 2024-11-20T21:29:03,303 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:03,303 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:03,303 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06f91812474a7664c203be2e75d69d0e: Waiting for close lock at 1732138143302Disabling compacts and flushes for region at 1732138143302Disabling writes for close at 1732138143303 (+1 ms)Writing region close event to WAL at 1732138143303Closed at 1732138143303 2024-11-20T21:29:03,305 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:29:03,310 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732138143305"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732138143305"}]},"ts":"1732138143305"} 2024-11-20T21:29:03,315 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:29:03,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:29:03,320 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732138143317"}]},"ts":"1732138143317"} 2024-11-20T21:29:03,324 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-20T21:29:03,325 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0b16ab3f13b7=0} racks are {/default-rack=0} 2024-11-20T21:29:03,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:29:03,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:29:03,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T21:29:03,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:29:03,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:29:03,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T21:29:03,327 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:29:03,327 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:29:03,327 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T21:29:03,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:29:03,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN}] 2024-11-20T21:29:03,331 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN 2024-11-20T21:29:03,333 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN; state=OFFLINE, location=0b16ab3f13b7,33007,1732138122755; forceNewPlan=false, retain=false 2024-11-20T21:29:03,486 INFO [0b16ab3f13b7:39535 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T21:29:03,487 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPENING, regionLocation=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:03,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN because future has completed 2024-11-20T21:29:03,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755}] 2024-11-20T21:29:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:29:03,650 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:03,651 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 06f91812474a7664c203be2e75d69d0e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:03,652 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,652 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:03,652 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,652 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,653 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,655 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf1 2024-11-20T21:29:03,655 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:03,658 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:03,658 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,660 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf2 2024-11-20T21:29:03,660 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:03,661 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:03,661 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,662 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,662 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,663 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,663 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,664 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T21:29:03,666 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,668 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:29:03,669 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 06f91812474a7664c203be2e75d69d0e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59001126, jitterRate=-0.12081471085548401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T21:29:03,669 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:03,669 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 06f91812474a7664c203be2e75d69d0e: Running coprocessor pre-open hook at 1732138143652Writing region info on filesystem at 1732138143652Initializing all the Stores at 1732138143653 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138143653Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138143653Cleaning up temporary data from old regions at 1732138143663 (+10 ms)Running coprocessor post-open hooks at 1732138143669 (+6 ms)Region opened successfully at 1732138143669 2024-11-20T21:29:03,671 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., pid=6, masterSystemTime=1732138143645 2024-11-20T21:29:03,675 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:03,675 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:03,676 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:03,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755 because future has completed 2024-11-20T21:29:03,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T21:29:03,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755 in 191 msec 2024-11-20T21:29:03,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T21:29:03,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN in 357 msec 2024-11-20T21:29:03,692 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:29:03,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732138143692"}]},"ts":"1732138143692"} 2024-11-20T21:29:03,696 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-20T21:29:03,697 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:29:03,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 841 msec 2024-11-20T21:29:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:29:04,011 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-20T21:29:04,011 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-20T21:29:04,013 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:29:04,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-20T21:29:04,018 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T21:29:04,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-20T21:29:04,035 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=2] 2024-11-20T21:29:04,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=06f91812474a7664c203be2e75d69d0e, source=0b16ab3f13b7,33007,1732138122755, destination=0b16ab3f13b7,45827,1732138122912, warming up region on 0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:29:04,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=06f91812474a7664c203be2e75d69d0e, source=0b16ab3f13b7,33007,1732138122755, destination=0b16ab3f13b7,45827,1732138122912, running balancer 2024-11-20T21:29:04,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57199, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:29:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE 2024-11-20T21:29:04,061 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE 2024-11-20T21:29:04,064 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=CLOSING, regionLocation=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:04,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE because future has completed 2024-11-20T21:29:04,069 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:29:04,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755}] 2024-11-20T21:29:04,070 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-20T21:29:04,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(7855): Warmup {ENCODED => 06f91812474a7664c203be2e75d69d0e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:04,083 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,085 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf1 2024-11-20T21:29:04,085 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:04,086 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:04,086 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,087 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf2 2024-11-20T21:29:04,087 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:04,087 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(1722): Closing 06f91812474a7664c203be2e75d69d0e, disabling compactions & flushes 2024-11-20T21:29:04,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. after waiting 0 ms 2024-11-20T21:29:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] regionserver.HRegion(1676): Region close journal for 06f91812474a7664c203be2e75d69d0e: Waiting for close lock at 1732138144088Disabling compacts and flushes for region at 1732138144088Disabling writes for close at 1732138144088Writing region close event to WAL at 1732138144096 (+8 ms)Closed at 1732138144096 2024-11-20T21:29:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T21:29:04,230 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,230 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:29:04,231 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 06f91812474a7664c203be2e75d69d0e, disabling compactions & flushes 2024-11-20T21:29:04,231 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,231 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,231 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. after waiting 0 ms 2024-11-20T21:29:04,231 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,231 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 06f91812474a7664c203be2e75d69d0e 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-20T21:29:04,251 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf1/c2360ae9649d42d1ba6aa93f5b251283 is 35, key is r1/cf1:q/1732138144038/Put/seqid=0 2024-11-20T21:29:04,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741887_1065 (size=4783) 2024-11-20T21:29:04,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741887_1065 (size=4783) 2024-11-20T21:29:04,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741887_1065 (size=4783) 2024-11-20T21:29:04,264 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf1/c2360ae9649d42d1ba6aa93f5b251283 2024-11-20T21:29:04,273 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf1/c2360ae9649d42d1ba6aa93f5b251283 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283 2024-11-20T21:29:04,280 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283, entries=1, sequenceid=5, filesize=4.7 K 2024-11-20T21:29:04,282 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 06f91812474a7664c203be2e75d69d0e in 51ms, sequenceid=5, compaction requested=false 2024-11-20T21:29:04,287 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T21:29:04,290 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,290 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 06f91812474a7664c203be2e75d69d0e: Waiting for close lock at 1732138144231Running coprocessor pre-close hooks at 1732138144231Disabling compacts and flushes for region at 1732138144231Disabling writes for close at 1732138144231Obtaining lock to block concurrent updates at 1732138144231Preparing flush snapshotting stores in 06f91812474a7664c203be2e75d69d0e at 1732138144231Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732138144231Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. at 1732138144232 (+1 ms)Flushing 06f91812474a7664c203be2e75d69d0e/cf1: creating writer at 1732138144232Flushing 06f91812474a7664c203be2e75d69d0e/cf1: appending metadata at 1732138144250 (+18 ms)Flushing 06f91812474a7664c203be2e75d69d0e/cf1: closing flushed file at 1732138144250Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@224ece95: reopening flushed file at 1732138144272 (+22 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 06f91812474a7664c203be2e75d69d0e in 51ms, sequenceid=5, compaction requested=false at 1732138144282 (+10 ms)Writing region close event to WAL at 1732138144283 (+1 ms)Running coprocessor post-close hooks at 1732138144288 (+5 ms)Closed at 1732138144290 (+2 ms) 2024-11-20T21:29:04,293 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 06f91812474a7664c203be2e75d69d0e move to 0b16ab3f13b7,45827,1732138122912 record at close sequenceid=5 2024-11-20T21:29:04,296 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,297 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=CLOSED 2024-11-20T21:29:04,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755 because future has completed 2024-11-20T21:29:04,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T21:29:04,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755 in 238 msec 2024-11-20T21:29:04,317 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE; state=CLOSED, location=0b16ab3f13b7,45827,1732138122912; forceNewPlan=false, retain=false 2024-11-20T21:29:04,468 INFO [0b16ab3f13b7:39535 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T21:29:04,469 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPENING, regionLocation=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:04,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE because future has completed 2024-11-20T21:29:04,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912}] 2024-11-20T21:29:04,629 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,629 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 06f91812474a7664c203be2e75d69d0e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:04,630 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,630 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:04,630 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,630 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,632 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,633 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf1 2024-11-20T21:29:04,633 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:04,638 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283 2024-11-20T21:29:04,639 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:04,639 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,639 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf2 2024-11-20T21:29:04,640 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:04,640 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:04,640 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,641 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,642 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,643 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,643 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,644 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T21:29:04,645 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,646 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 06f91812474a7664c203be2e75d69d0e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64467898, jitterRate=-0.03935346007347107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T21:29:04,646 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:04,647 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 06f91812474a7664c203be2e75d69d0e: Running coprocessor pre-open hook at 1732138144630Writing region info on filesystem at 1732138144630Initializing all the Stores at 1732138144631 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138144631Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138144632 (+1 ms)Cleaning up temporary data from old regions at 1732138144643 (+11 ms)Running coprocessor post-open hooks at 1732138144646 (+3 ms)Region opened successfully at 1732138144647 (+1 ms) 2024-11-20T21:29:04,648 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., pid=9, masterSystemTime=1732138144625 2024-11-20T21:29:04,651 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,651 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,651 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPEN, openSeqNum=9, regionLocation=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:04,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912 because future has completed 2024-11-20T21:29:04,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T21:29:04,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912 in 183 msec 2024-11-20T21:29:04,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE in 599 msec 2024-11-20T21:29:04,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:29:04,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:29:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:43352 deadline: 1732138204672, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=45827 startCode=1732138122912. As of locationSeqNum=5. 2024-11-20T21:29:04,694 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=45827 startCode=1732138122912. As of locationSeqNum=5. 2024-11-20T21:29:04,694 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=45827 startCode=1732138122912. As of locationSeqNum=5. 2024-11-20T21:29:04,694 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,45827,1732138122912, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=45827 startCode=1732138122912. As of locationSeqNum=5. 2024-11-20T21:29:04,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:29:04,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:29:04,821 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06f91812474a7664c203be2e75d69d0e 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-20T21:29:04,845 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e is 29, key is r1/cf1:/1732138144812/DeleteFamily/seqid=0 2024-11-20T21:29:04,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741888_1066 (size=4906) 2024-11-20T21:29:04,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741888_1066 (size=4906) 2024-11-20T21:29:04,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741888_1066 (size=4906) 2024-11-20T21:29:04,856 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e 2024-11-20T21:29:04,865 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fcc2c3d6b1e48bab68ce0d67c36777e 2024-11-20T21:29:04,888 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf2/4b604dc217fb43eeb9c906729a516093 is 29, key is r1/cf2:/1732138144812/DeleteFamily/seqid=0 2024-11-20T21:29:04,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741889_1067 (size=4906) 2024-11-20T21:29:04,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741889_1067 (size=4906) 2024-11-20T21:29:04,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741889_1067 (size=4906) 2024-11-20T21:29:04,904 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf2/4b604dc217fb43eeb9c906729a516093 2024-11-20T21:29:04,911 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4b604dc217fb43eeb9c906729a516093 2024-11-20T21:29:04,912 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e 2024-11-20T21:29:04,926 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fcc2c3d6b1e48bab68ce0d67c36777e 2024-11-20T21:29:04,926 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T21:29:04,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf2/4b604dc217fb43eeb9c906729a516093 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/4b604dc217fb43eeb9c906729a516093 2024-11-20T21:29:04,940 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4b604dc217fb43eeb9c906729a516093 2024-11-20T21:29:04,940 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/4b604dc217fb43eeb9c906729a516093, entries=1, sequenceid=12, filesize=4.8 K 2024-11-20T21:29:04,942 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 06f91812474a7664c203be2e75d69d0e in 121ms, sequenceid=12, compaction requested=false 2024-11-20T21:29:04,942 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 06f91812474a7664c203be2e75d69d0e: 2024-11-20T21:29:04,945 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T21:29:04,947 DEBUG [Time-limited test {}] regionserver.HStore(1541): 06f91812474a7664c203be2e75d69d0e/cf1 is initiating major compaction (all files) 2024-11-20T21:29:04,948 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:29:04,948 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:29:04,948 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 06f91812474a7664c203be2e75d69d0e/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:04,950 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283, hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e] into tmpdir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp, totalSize=9.5 K 2024-11-20T21:29:04,951 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c2360ae9649d42d1ba6aa93f5b251283, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732138144038 2024-11-20T21:29:04,952 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6fcc2c3d6b1e48bab68ce0d67c36777e, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T21:29:04,987 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 06f91812474a7664c203be2e75d69d0e#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:29:05,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741890_1068 (size=4626) 2024-11-20T21:29:05,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741890_1068 (size=4626) 2024-11-20T21:29:05,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741890_1068 (size=4626) 2024-11-20T21:29:05,031 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf1/0bb20fb64e7a4855aee6b5e33ab5a653 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/0bb20fb64e7a4855aee6b5e33ab5a653 2024-11-20T21:29:05,046 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 06f91812474a7664c203be2e75d69d0e/cf1 of 06f91812474a7664c203be2e75d69d0e into 0bb20fb64e7a4855aee6b5e33ab5a653(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:29:05,046 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 06f91812474a7664c203be2e75d69d0e: 2024-11-20T21:29:05,047 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-20T21:29:05,047 DEBUG [Time-limited test {}] regionserver.HStore(1541): 06f91812474a7664c203be2e75d69d0e/cf2 is initiating major compaction (all files) 2024-11-20T21:29:05,047 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:29:05,047 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:29:05,047 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 06f91812474a7664c203be2e75d69d0e/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,047 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/4b604dc217fb43eeb9c906729a516093] into tmpdir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp, totalSize=4.8 K 2024-11-20T21:29:05,048 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4b604dc217fb43eeb9c906729a516093, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-20T21:29:05,058 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 06f91812474a7664c203be2e75d69d0e#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:29:05,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741891_1069 (size=4592) 2024-11-20T21:29:05,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741891_1069 (size=4592) 2024-11-20T21:29:05,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741891_1069 (size=4592) 2024-11-20T21:29:05,095 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/.tmp/cf2/2c5652ebaeb74919ac2e60f4d3a21712 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/2c5652ebaeb74919ac2e60f4d3a21712 2024-11-20T21:29:05,102 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 06f91812474a7664c203be2e75d69d0e/cf2 of 06f91812474a7664c203be2e75d69d0e into 2c5652ebaeb74919ac2e60f4d3a21712(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:29:05,102 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 06f91812474a7664c203be2e75d69d0e: 2024-11-20T21:29:05,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=06f91812474a7664c203be2e75d69d0e, source=0b16ab3f13b7,45827,1732138122912, destination=0b16ab3f13b7,33007,1732138122755, warming up region on 0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:05,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=06f91812474a7664c203be2e75d69d0e, source=0b16ab3f13b7,45827,1732138122912, destination=0b16ab3f13b7,33007,1732138122755, running balancer 2024-11-20T21:29:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE 2024-11-20T21:29:05,108 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE 2024-11-20T21:29:05,109 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=CLOSING, regionLocation=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:05,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(7855): Warmup {ENCODED => 06f91812474a7664c203be2e75d69d0e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:05,110 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,111 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf1 2024-11-20T21:29:05,112 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:05,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE because future has completed 2024-11-20T21:29:05,113 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:29:05,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912}] 2024-11-20T21:29:05,119 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/0bb20fb64e7a4855aee6b5e33ab5a653 2024-11-20T21:29:05,129 INFO [StoreFileOpener-06f91812474a7664c203be2e75d69d0e-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fcc2c3d6b1e48bab68ce0d67c36777e 2024-11-20T21:29:05,129 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e 2024-11-20T21:29:05,134 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283 2024-11-20T21:29:05,134 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:05,134 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,136 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf2 2024-11-20T21:29:05,136 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:05,143 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/2c5652ebaeb74919ac2e60f4d3a21712 2024-11-20T21:29:05,148 INFO [StoreFileOpener-06f91812474a7664c203be2e75d69d0e-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4b604dc217fb43eeb9c906729a516093 2024-11-20T21:29:05,149 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/4b604dc217fb43eeb9c906729a516093 2024-11-20T21:29:05,149 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(1722): Closing 06f91812474a7664c203be2e75d69d0e, disabling compactions & flushes 2024-11-20T21:29:05,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. after waiting 0 ms 2024-11-20T21:29:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.HRegion(1676): Region close journal for 06f91812474a7664c203be2e75d69d0e: Waiting for close lock at 1732138145149Disabling compacts and flushes for region at 1732138145149Disabling writes for close at 1732138145149Writing region close event to WAL at 1732138145153 (+4 ms)Closed at 1732138145153 2024-11-20T21:29:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-20T21:29:05,267 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,267 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:29:05,267 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 06f91812474a7664c203be2e75d69d0e, disabling compactions & flushes 2024-11-20T21:29:05,267 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,267 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,267 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. after waiting 0 ms 2024-11-20T21:29:05,267 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,268 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283, hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e] to archive 2024-11-20T21:29:05,272 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:29:05,277 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283 to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/c2360ae9649d42d1ba6aa93f5b251283 2024-11-20T21:29:05,279 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/6fcc2c3d6b1e48bab68ce0d67c36777e 2024-11-20T21:29:05,296 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/4b604dc217fb43eeb9c906729a516093] to archive 2024-11-20T21:29:05,297 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:29:05,301 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/4b604dc217fb43eeb9c906729a516093 to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/4b604dc217fb43eeb9c906729a516093 2024-11-20T21:29:05,313 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-20T21:29:05,314 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,314 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 06f91812474a7664c203be2e75d69d0e: Waiting for close lock at 1732138145267Running coprocessor pre-close hooks at 1732138145267Disabling compacts and flushes for region at 1732138145267Disabling writes for close at 1732138145267Writing region close event to WAL at 1732138145302 (+35 ms)Running coprocessor post-close hooks at 1732138145314 (+12 ms)Closed at 1732138145314 2024-11-20T21:29:05,314 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 06f91812474a7664c203be2e75d69d0e move to 0b16ab3f13b7,33007,1732138122755 record at close sequenceid=12 2024-11-20T21:29:05,317 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,321 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=CLOSED 2024-11-20T21:29:05,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912 because future has completed 2024-11-20T21:29:05,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T21:29:05,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912 in 216 msec 2024-11-20T21:29:05,334 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE; state=CLOSED, location=0b16ab3f13b7,33007,1732138122755; forceNewPlan=false, retain=false 2024-11-20T21:29:05,485 INFO [0b16ab3f13b7:39535 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T21:29:05,485 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPENING, regionLocation=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:05,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE because future has completed 2024-11-20T21:29:05,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755}] 2024-11-20T21:29:05,644 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,645 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 06f91812474a7664c203be2e75d69d0e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:05,645 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,645 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:05,645 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,645 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,647 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,648 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf1 2024-11-20T21:29:05,648 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:05,654 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/0bb20fb64e7a4855aee6b5e33ab5a653 2024-11-20T21:29:05,654 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:05,654 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,655 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf2 2024-11-20T21:29:05,655 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:05,661 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/2c5652ebaeb74919ac2e60f4d3a21712 2024-11-20T21:29:05,661 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:05,661 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,662 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,663 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,664 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,664 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,665 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T21:29:05,666 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,667 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 06f91812474a7664c203be2e75d69d0e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68937256, jitterRate=0.027245163917541504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T21:29:05,668 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,668 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 06f91812474a7664c203be2e75d69d0e: Running coprocessor pre-open hook at 1732138145645Writing region info on filesystem at 1732138145645Initializing all the Stores at 1732138145646 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138145646Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138145647 (+1 ms)Cleaning up temporary data from old regions at 1732138145664 (+17 ms)Running coprocessor post-open hooks at 1732138145668 (+4 ms)Region opened successfully at 1732138145668 2024-11-20T21:29:05,669 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., pid=12, masterSystemTime=1732138145641 2024-11-20T21:29:05,671 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,671 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,672 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPEN, openSeqNum=18, regionLocation=0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:05,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755 because future has completed 2024-11-20T21:29:05,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-20T21:29:05,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,33007,1732138122755 in 187 msec 2024-11-20T21:29:05,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, REOPEN/MOVE in 572 msec 2024-11-20T21:29:05,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:29:05,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:29:05,713 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0b16ab3f13b7,33007,1732138122755: testing ***** 2024-11-20T21:29:05,713 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T21:29:05,715 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T21:29:05,716 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T21:29:05,720 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T21:29:05,722 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T21:29:05,732 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 491401456 }, "NonHeapMemoryUsage": { "committed": 172228608, "init": 7667712, "max": -1, "used": 169497672 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0b16ab3f13b7", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 46, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 4, "ProcessCallTime_95th_percentile": 36, "ProcessCallTime_98th_percentile": 46, "ProcessCallTime_99th_percentile": 46, "ProcessCallTime_99.9th_percentile": 46, "ProcessCallTime_TimeRangeCount_0-1": 11, "ProcessCallTime_TimeRangeCount_1-3": 13, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_30-100": 2, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 2, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 2, "QueueCallTime_99th_percentile": 2, "QueueCallTime_99.9th_percentile": 2, "QueueCallTime_TimeRangeCount_0-1": 30, "QueueCallTime_TimeRangeCount_1-3": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 46, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 4, "TotalCallTime_95th_percentile": 37, "TotalCallTime_98th_percentile": 46, "TotalCallTime_99th_percentile": 46, "TotalCallTime_99.9th_percentile": 46, "TotalCallTime_TimeRangeCount_0-1": 10, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 6, "TotalCallTime_TimeRangeCount_30-100": 2, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0b16ab3f13b7", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56682", "tag.serverName": "0b16ab3f13b7,45827,1732138122912", "tag.clusterId": "e28df62c-830d-4222-9e72-1212682cc572", "tag.Context": "regionserver", "tag.Hostname": "0b16ab3f13b7", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732138122912, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 55, "MajorCompactionTime_max": 93, "MajorCompactionTime_mean": 74, "MajorCompactionTime_25th_percentile": 55, "MajorCompactionTime_median": 56, "MajorCompactionTime_75th_percentile": 93, "MajorCompactionTime_90th_percentile": 93, "MajorCompactionTime_95th_percentile": 93, "MajorCompactionTime_98th_percentile": 93, "MajorCompactionTime_99th_percentile": 93, "MajorCompactionTime_99.9th_percentile": 93, "MajorCompactionTime_TimeRangeCount_30-100": 2, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 3, "Delete_max": 3, "Delete_mean": 3, "Delete_25th_percentile": 3, "Delete_median": 3, "Delete_75th_percentile": 3, "Delete_90th_percentile": 3, "Delete_95th_percentile": 3, "Delete_98th_percentile": 3, "Delete_99th_percentile": 3, "Delete_99.9th_percentile": 3, "Delete_TimeRangeCount_1-3": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 55, "CompactionTime_max": 93, "CompactionTime_mean": 74, "CompactionTime_25th_percentile": 55, "CompactionTime_median": 56, "CompactionTime_75th_percentile": 93, "CompactionTime_90th_percentile": 93, "CompactionTime_95th_percentile": 93, "CompactionTime_98th_percentile": 93, "CompactionTime_99th_percentile": 93, "CompactionTime_99.9th_percentile": 93, "CompactionTime_TimeRangeCount_30-100": 2, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 121, "FlushTime_max": 121, "FlushTime_mean": 121, "FlushTime_25th_percentile": 121, "FlushTime_median": 121, "FlushTime_75th_percentile": 121, "FlushTime_90th_percentile": 121, "FlushTime_95th_percentile": 121, "FlushTime_98th_percentile": 121, "FlushTime_99th_percentile": 121, "FlushTime_99.9th_percentile": 121, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T21:29:05,735 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39535 {}] master.MasterRpcServices(700): 0b16ab3f13b7,33007,1732138122755 reported a fatal error: ***** ABORTING region server 0b16ab3f13b7,33007,1732138122755: testing ***** 2024-11-20T21:29:05,737 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b16ab3f13b7,33007,1732138122755' ***** 2024-11-20T21:29:05,737 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-20T21:29:05,738 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:29:05,738 INFO [RS:0;0b16ab3f13b7:33007 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T21:29:05,738 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:29:05,738 INFO [RS:0;0b16ab3f13b7:33007 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T21:29:05,738 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45827 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:52642 deadline: 1732138205738, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=33007 startCode=1732138122755. As of locationSeqNum=12. 2024-11-20T21:29:05,739 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(956): aborting server 0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:05,739 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:29:05,739 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,45827,1732138122912, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,45827,1732138122912, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=33007 startCode=1732138122755. As of locationSeqNum=12. 2024-11-20T21:29:05,739 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,45827,1732138122912, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=33007 startCode=1732138122755. As of locationSeqNum=12. 2024-11-20T21:29:05,739 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,45827,1732138122912, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0b16ab3f13b7 port=33007 startCode=1732138122755. As of locationSeqNum=12. 2024-11-20T21:29:05,739 INFO [RS:0;0b16ab3f13b7:33007 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b16ab3f13b7:33007. 2024-11-20T21:29:05,739 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06f91812474a7664c203be2e75d69d0e, disabling compactions & flushes 2024-11-20T21:29:05,739 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,739 DEBUG [RS:0;0b16ab3f13b7:33007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:29:05,739 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,739 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. after waiting 0 ms 2024-11-20T21:29:05,739 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,740 DEBUG [RS:0;0b16ab3f13b7:33007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:05,740 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:29:05,740 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:29:05,740 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:29:05,740 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T21:29:05,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0b16ab3f13b7,33007,1732138122755 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:05,741 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T21:29:05,741 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1325): Online Regions={06f91812474a7664c203be2e75d69d0e=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T21:29:05,741 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:29:05,741 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:29:05,741 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:29:05,741 DEBUG [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1351): Waiting on 06f91812474a7664c203be2e75d69d0e, 1588230740 2024-11-20T21:29:05,741 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:29:05,741 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:29:05,742 ERROR [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-20T21:29:05,742 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:29:05,742 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0b16ab3f13b7,33007,1732138122755 aborting 2024-11-20T21:29:05,742 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0b16ab3f13b7,33007,1732138122755 aborting 2024-11-20T21:29:05,742 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 from cache 2024-11-20T21:29:05,742 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,742 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06f91812474a7664c203be2e75d69d0e: Waiting for close lock at 1732138145739Running coprocessor pre-close hooks at 1732138145739Disabling compacts and flushes for region at 1732138145739Disabling writes for close at 1732138145739Writing region close event to WAL at 1732138145742 (+3 ms)Running coprocessor post-close hooks at 1732138145742Closed at 1732138145742 2024-11-20T21:29:05,742 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:29:05,743 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732138145741Running coprocessor pre-close hooks at 1732138145741Disabling compacts and flushes for region at 1732138145741Disabling writes for close at 1732138145741Writing region close event to WAL at 1732138145742 (+1 ms)Running coprocessor post-close hooks at 1732138145742Closed at 1732138145742 2024-11-20T21:29:05,743 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:05,743 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T21:29:05,775 INFO [regionserver/0b16ab3f13b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T21:29:05,776 INFO [regionserver/0b16ab3f13b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T21:29:05,816 INFO [regionserver/0b16ab3f13b7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:29:05,849 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:29:05,851 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1] 2024-11-20T21:29:05,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0b16ab3f13b7,33007,1732138122755 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:05,852 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0b16ab3f13b7,33007,1732138122755 aborting 2024-11-20T21:29:05,852 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0b16ab3f13b7,33007,1732138122755 aborting 2024-11-20T21:29:05,852 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 from cache 2024-11-20T21:29:05,942 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(976): stopping server 0b16ab3f13b7,33007,1732138122755; all regions closed. 2024-11-20T21:29:05,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741836_1012 (size=7323) 2024-11-20T21:29:05,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741836_1012 (size=7323) 2024-11-20T21:29:05,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741836_1012 (size=7323) 2024-11-20T21:29:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741834_1010 (size=2288) 2024-11-20T21:29:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741834_1010 (size=2288) 2024-11-20T21:29:05,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741834_1010 (size=2288) 2024-11-20T21:29:05,959 DEBUG [RS:0;0b16ab3f13b7:33007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:05,959 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:29:05,959 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:29:05,959 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.ChoreService(370): Chore service for: regionserver/0b16ab3f13b7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T21:29:05,959 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:29:05,959 INFO [regionserver/0b16ab3f13b7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:29:05,960 INFO [RS:0;0b16ab3f13b7:33007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33007 2024-11-20T21:29:05,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:29:05,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b16ab3f13b7,33007,1732138122755 2024-11-20T21:29:05,967 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:29:05,970 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b16ab3f13b7,33007,1732138122755] 2024-11-20T21:29:05,971 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b16ab3f13b7,33007,1732138122755 already deleted, retry=false 2024-11-20T21:29:05,972 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0b16ab3f13b7,33007,1732138122755 on 0b16ab3f13b7,39535,1732138121558 2024-11-20T21:29:05,978 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0b16ab3f13b7,33007,1732138122755, splitWal=true, meta=true 2024-11-20T21:29:05,981 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 0b16ab3f13b7,33007,1732138122755 (carryingMeta=true) 0b16ab3f13b7,33007,1732138122755/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@63042263[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T21:29:05,981 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0b16ab3f13b7,33007,1732138122755, splitWal=true, meta=true 2024-11-20T21:29:05,983 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 0b16ab3f13b7,33007,1732138122755, splitWal=true, meta=true, isMeta: true 2024-11-20T21:29:05,984 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting 2024-11-20T21:29:05,986 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0b16ab3f13b7,33007,1732138122755 WAL count=1, meta=true 2024-11-20T21:29:05,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta}] 2024-11-20T21:29:05,994 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:05,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, worker=0b16ab3f13b7,40171,1732138123009}] 2024-11-20T21:29:06,059 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:29:06,061 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1] 2024-11-20T21:29:06,063 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0b16ab3f13b7:33007 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0b16ab3f13b7/172.17.0.3:33007 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:06,063 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1, error=java.net.ConnectException: Call to address=0b16ab3f13b7:33007 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0b16ab3f13b7/172.17.0.3:33007 2024-11-20T21:29:06,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-20T21:29:06,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 from cache 2024-11-20T21:29:06,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 0b16ab3f13b7:33007 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0b16ab3f13b7/172.17.0.3:33007 2024-11-20T21:29:06,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:06,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x1005f1840a00001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:06,071 INFO [RS:0;0b16ab3f13b7:33007 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:29:06,071 INFO [RS:0;0b16ab3f13b7:33007 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b16ab3f13b7,33007,1732138122755; zookeeper connection closed. 2024-11-20T21:29:06,071 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66074d8a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66074d8a 2024-11-20T21:29:06,157 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:29:06,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52637, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:29:06,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40171 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-20T21:29:06,192 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, size=7.2 K (7323bytes) 2024-11-20T21:29:06,192 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta 2024-11-20T21:29:06,193 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta after 1ms 2024-11-20T21:29:06,237 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:06,237 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta took 46ms 2024-11-20T21:29:06,252 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T21:29:06,258 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta so closing down 2024-11-20T21:29:06,259 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:06,267 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta.temp 2024-11-20T21:29:06,311 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta.temp 2024-11-20T21:29:06,312 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:06,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741892_1070 (size=7071) 2024-11-20T21:29:06,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741892_1070 (size=7071) 2024-11-20T21:29:06,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741892_1070 (size=7071) 2024-11-20T21:29:06,365 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:06,367 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta.temp to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T21:29:06,367 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 125 ms; skipped=1; WAL=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-20T21:29:06,367 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, journal: Splitting hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, size=7.2 K (7323bytes) at 1732138146192Finishing writing output for hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta so closing down at 1732138146259 (+67 ms)Creating recovered edits writer path=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta.temp at 1732138146311 (+52 ms)3 split writer threads finished at 1732138146312 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732138146365 (+53 ms)Rename recovered edits hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta.temp to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732138146367 (+2 ms)Processed 16 edits across 1 Regions in 125 ms; skipped=1; WAL=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1732138146367 2024-11-20T21:29:06,367 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta 2024-11-20T21:29:06,369 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-20T21:29:06,370 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:29:06,371 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1] 2024-11-20T21:29:06,371 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0b16ab3f13b7:33007 this server is in the failed servers list 2024-11-20T21:29:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-20T21:29:06,372 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0b16ab3f13b7:33007 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0b16ab3f13b7:33007 2024-11-20T21:29:06,372 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0b16ab3f13b7:33007 2024-11-20T21:29:06,372 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 from cache 2024-11-20T21:29:06,377 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs 2024-11-20T21:29:06,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T21:29:06,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, worker=0b16ab3f13b7,40171,1732138123009 in 382 msec 2024-11-20T21:29:06,385 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:06,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T21:29:06,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 0b16ab3f13b7%2C33007%2C1732138122755.meta.1732138126381.meta, worker=0b16ab3f13b7,40171,1732138123009 in 398 msec 2024-11-20T21:29:06,393 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting dir is empty, no logs to split. 2024-11-20T21:29:06,393 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0b16ab3f13b7,33007,1732138122755 WAL count=0, meta=true 2024-11-20T21:29:06,393 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0b16ab3f13b7,33007,1732138122755 WAL splitting is done? wals=0, meta=true 2024-11-20T21:29:06,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T21:29:06,398 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:29:06,399 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T21:29:06,550 DEBUG [0b16ab3f13b7:39535 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T21:29:06,550 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(204): Hosts are {0b16ab3f13b7=0} racks are {/default-rack=0} 2024-11-20T21:29:06,550 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:29:06,550 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:29:06,550 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:29:06,550 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:29:06,550 INFO [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:29:06,551 INFO [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:29:06,551 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:29:06,551 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:06,553 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b16ab3f13b7,40171,1732138123009, state=OPENING 2024-11-20T21:29:06,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:29:06,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:29:06,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:29:06,555 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:29:06,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:29:06,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:29:06,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,40171,1732138123009}] 2024-11-20T21:29:06,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:29:06,714 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T21:29:06,714 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:06,714 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T21:29:06,716 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C40171%2C1732138123009.meta, suffix=.meta, logDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,40171,1732138123009, archiveDir=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs, maxLogs=32 2024-11-20T21:29:06,735 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,40171,1732138123009/0b16ab3f13b7%2C40171%2C1732138123009.meta.1732138146717.meta, exclude list is [], retry=0 2024-11-20T21:29:06,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:06,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:06,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:06,772 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,40171,1732138123009/0b16ab3f13b7%2C40171%2C1732138123009.meta.1732138146717.meta 2024-11-20T21:29:06,772 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:06,772 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:06,773 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T21:29:06,773 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T21:29:06,773 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T21:29:06,773 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T21:29:06,773 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:06,774 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T21:29:06,774 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T21:29:06,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T21:29:06,777 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:29:06,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:06,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:29:06,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T21:29:06,779 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:29:06,779 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:06,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:29:06,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T21:29:06,782 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:29:06,782 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:06,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:29:06,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T21:29:06,785 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:29:06,786 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:06,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:29:06,786 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:29:06,787 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740 2024-11-20T21:29:06,789 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740 2024-11-20T21:29:06,789 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T21:29:06,828 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:06,831 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T21:29:06,832 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-20T21:29:06,868 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/info/0cfb7acbaa0d451e989aea5ad3ab0dd0 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e./info:regioninfo/1732138145672/Put/seqid=0 2024-11-20T21:29:06,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741894_1072 (size=11177) 2024-11-20T21:29:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741894_1072 (size=11177) 2024-11-20T21:29:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741894_1072 (size=11177) 2024-11-20T21:29:06,876 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/info/0cfb7acbaa0d451e989aea5ad3ab0dd0 2024-11-20T21:29:06,879 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:29:06,881 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,40171,1732138123009, seqNum=-1] 2024-11-20T21:29:06,881 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:29:06,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:29:06,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40171 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Get size: 119 connection: 172.17.0.3:55584 deadline: 1732138206883, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:06,885 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,40171,1732138123009, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,40171,1732138123009, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 0b16ab3f13b7,40171,1732138123009 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:29:06,885 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,40171,1732138123009, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 0b16ab3f13b7,40171,1732138123009 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:29:06,885 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,40171,1732138123009, seqNum=-1 from cache 2024-11-20T21:29:06,906 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/ns/da5d662eab4f4f6d86f9fa65378a53a7 is 43, key is default/ns:d/1732138126716/Put/seqid=0 2024-11-20T21:29:06,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741895_1073 (size=5153) 2024-11-20T21:29:06,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741895_1073 (size=5153) 2024-11-20T21:29:06,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741895_1073 (size=5153) 2024-11-20T21:29:06,914 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/ns/da5d662eab4f4f6d86f9fa65378a53a7 2024-11-20T21:29:06,936 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/table/82a1e58c0bb4441080c636f7c7e85b11 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732138143692/Put/seqid=0 2024-11-20T21:29:06,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741896_1074 (size=5431) 2024-11-20T21:29:06,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741896_1074 (size=5431) 2024-11-20T21:29:06,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741896_1074 (size=5431) 2024-11-20T21:29:06,944 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/table/82a1e58c0bb4441080c636f7c7e85b11 2024-11-20T21:29:06,951 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/info/0cfb7acbaa0d451e989aea5ad3ab0dd0 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/info/0cfb7acbaa0d451e989aea5ad3ab0dd0 2024-11-20T21:29:06,962 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/info/0cfb7acbaa0d451e989aea5ad3ab0dd0, entries=36, sequenceid=18, filesize=10.9 K 2024-11-20T21:29:06,963 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/ns/da5d662eab4f4f6d86f9fa65378a53a7 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/ns/da5d662eab4f4f6d86f9fa65378a53a7 2024-11-20T21:29:06,971 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/ns/da5d662eab4f4f6d86f9fa65378a53a7, entries=2, sequenceid=18, filesize=5.0 K 2024-11-20T21:29:06,972 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/table/82a1e58c0bb4441080c636f7c7e85b11 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/table/82a1e58c0bb4441080c636f7c7e85b11 2024-11-20T21:29:06,979 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/table/82a1e58c0bb4441080c636f7c7e85b11, entries=2, sequenceid=18, filesize=5.3 K 2024-11-20T21:29:06,980 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 148ms, sequenceid=18, compaction requested=false; wal=null 2024-11-20T21:29:06,981 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-20T21:29:06,983 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:29:06,983 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:29:06,984 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:29:06,986 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:29:06,992 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T21:29:06,994 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74015104, jitterRate=0.10291099548339844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:29:06,994 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T21:29:06,995 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732138146774Writing region info on filesystem at 1732138146774Initializing all the Stores at 1732138146775 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138146775Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138146775Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138146775Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732138146775Obtaining lock to block concurrent updates at 1732138146832 (+57 ms)Preparing flush snapshotting stores in 1588230740 at 1732138146832Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732138146832Flushing stores of hbase:meta,,1.1588230740 at 1732138146832Flushing 1588230740/info: creating writer at 1732138146832Flushing 1588230740/info: appending metadata at 1732138146867 (+35 ms)Flushing 1588230740/info: closing flushed file at 1732138146867Flushing 1588230740/ns: creating writer at 1732138146884 (+17 ms)Flushing 1588230740/ns: appending metadata at 1732138146905 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732138146905Flushing 1588230740/table: creating writer at 1732138146921 (+16 ms)Flushing 1588230740/table: appending metadata at 1732138146936 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732138146936Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@107fe4f9: reopening flushed file at 1732138146950 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23c03d48: reopening flushed file at 1732138146962 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30ef8ac7: reopening flushed file at 1732138146971 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 148ms, sequenceid=18, compaction requested=false; wal=null at 1732138146980 (+9 ms)Cleaning up temporary data from old regions at 1732138146983 (+3 ms)Running coprocessor post-open hooks at 1732138146994 (+11 ms)Region opened successfully at 1732138146995 (+1 ms) 2024-11-20T21:29:06,997 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732138146709 2024-11-20T21:29:07,002 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T21:29:07,002 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T21:29:07,005 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:07,009 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b16ab3f13b7,40171,1732138123009, state=OPEN 2024-11-20T21:29:07,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:29:07,017 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:07,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:29:07,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:29:07,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:29:07,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:29:07,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:29:07,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T21:29:07,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,40171,1732138123009 in 462 msec 2024-11-20T21:29:07,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-20T21:29:07,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 630 msec 2024-11-20T21:29:07,030 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 0b16ab3f13b7,33007,1732138122755 had 2 regions 2024-11-20T21:29:07,032 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0b16ab3f13b7,33007,1732138122755, splitWal=true, meta=true, isMeta: false 2024-11-20T21:29:07,034 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0b16ab3f13b7,33007,1732138122755 WAL count=1, meta=false 2024-11-20T21:29:07,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 0b16ab3f13b7%2C33007%2C1732138122755.1732138125949}] 2024-11-20T21:29:07,037 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:07,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, worker=0b16ab3f13b7,45827,1732138122912}] 2024-11-20T21:29:07,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45827 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-20T21:29:07,220 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, size=2.2 K (2288bytes) 2024-11-20T21:29:07,221 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 2024-11-20T21:29:07,221 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 after 0ms 2024-11-20T21:29:07,251 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:07,251 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 took 31ms 2024-11-20T21:29:07,253 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 06f91812474a7664c203be2e75d69d0e: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-20T21:29:07,254 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 so closing down 2024-11-20T21:29:07,254 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:07,254 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:07,254 INFO [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-20T21:29:07,254 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, journal: Splitting hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, size=2.2 K (2288bytes) at 1732138147221Finishing writing output for hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 so closing down at 1732138147254 (+33 ms)3 split writer threads finished at 1732138147254Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732138147254 2024-11-20T21:29:07,254 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 2024-11-20T21:29:07,254 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-20T21:29:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-20T21:29:07,258 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting/0b16ab3f13b7%2C33007%2C1732138122755.1732138125949 to hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs 2024-11-20T21:29:07,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-20T21:29:07,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, worker=0b16ab3f13b7,45827,1732138122912 in 219 msec 2024-11-20T21:29:07,262 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:07,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-20T21:29:07,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 0b16ab3f13b7%2C33007%2C1732138122755.1732138125949, worker=0b16ab3f13b7,45827,1732138122912 in 228 msec 2024-11-20T21:29:07,267 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,33007,1732138122755-splitting dir is empty, no logs to split. 2024-11-20T21:29:07,267 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0b16ab3f13b7,33007,1732138122755 WAL count=0, meta=false 2024-11-20T21:29:07,267 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 0b16ab3f13b7,33007,1732138122755 WAL splitting is done? wals=0, meta=false 2024-11-20T21:29:07,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN}] 2024-11-20T21:29:07,271 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN 2024-11-20T21:29:07,272 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T21:29:07,422 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(204): Hosts are {0b16ab3f13b7=0} racks are {/default-rack=0} 2024-11-20T21:29:07,422 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T21:29:07,422 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T21:29:07,422 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T21:29:07,422 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T21:29:07,422 INFO [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T21:29:07,422 INFO [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T21:29:07,422 DEBUG [0b16ab3f13b7:39535 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T21:29:07,423 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPENING, regionLocation=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:07,424 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0b16ab3f13b7:33007 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0b16ab3f13b7/172.17.0.3:33007 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:07,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1, error=java.net.ConnectException: Call to address=0b16ab3f13b7:33007 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0b16ab3f13b7/172.17.0.3:33007 2024-11-20T21:29:07,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T21:29:07,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,33007,1732138122755, seqNum=-1 from cache 2024-11-20T21:29:07,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 0b16ab3f13b7:33007 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0b16ab3f13b7/172.17.0.3:33007 2024-11-20T21:29:07,529 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:29:07,529 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,40171,1732138123009, seqNum=-1] 2024-11-20T21:29:07,529 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:29:07,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39801, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:29:07,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN because future has completed 2024-11-20T21:29:07,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912}] 2024-11-20T21:29:07,690 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:07,690 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 06f91812474a7664c203be2e75d69d0e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:07,690 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,690 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:07,690 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,690 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,692 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,693 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf1 2024-11-20T21:29:07,693 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:07,700 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf1/0bb20fb64e7a4855aee6b5e33ab5a653 2024-11-20T21:29:07,700 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:07,701 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,701 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06f91812474a7664c203be2e75d69d0e columnFamilyName cf2 2024-11-20T21:29:07,701 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:07,708 DEBUG [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/cf2/2c5652ebaeb74919ac2e60f4d3a21712 2024-11-20T21:29:07,708 INFO [StoreOpener-06f91812474a7664c203be2e75d69d0e-1 {}] regionserver.HStore(327): Store=06f91812474a7664c203be2e75d69d0e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:07,708 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,709 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,711 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,712 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,712 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,713 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-20T21:29:07,715 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,716 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 06f91812474a7664c203be2e75d69d0e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70271754, jitterRate=0.047130733728408813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-20T21:29:07,716 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:07,717 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 06f91812474a7664c203be2e75d69d0e: Running coprocessor pre-open hook at 1732138147691Writing region info on filesystem at 1732138147691Initializing all the Stores at 1732138147691Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138147691Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138147692 (+1 ms)Cleaning up temporary data from old regions at 1732138147712 (+20 ms)Running coprocessor post-open hooks at 1732138147716 (+4 ms)Region opened successfully at 1732138147717 (+1 ms) 2024-11-20T21:29:07,725 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., pid=21, masterSystemTime=1732138147686 2024-11-20T21:29:07,727 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:07,728 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=06f91812474a7664c203be2e75d69d0e, regionState=OPEN, openSeqNum=18, regionLocation=0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:07,728 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:07,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912 because future has completed 2024-11-20T21:29:07,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-20T21:29:07,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 06f91812474a7664c203be2e75d69d0e, server=0b16ab3f13b7,45827,1732138122912 in 203 msec 2024-11-20T21:29:07,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-20T21:29:07,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=06f91812474a7664c203be2e75d69d0e, ASSIGN in 471 msec 2024-11-20T21:29:07,745 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 0b16ab3f13b7,33007,1732138122755 after splitting done 2024-11-20T21:29:07,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 0b16ab3f13b7,33007,1732138122755, splitWal=true, meta=true in 1.7730 sec 2024-11-20T21:29:07,889 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:29:07,890 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,40171,1732138123009, seqNum=-1] 2024-11-20T21:29:08,010 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0b16ab3f13b7:33007 this server is in the failed servers list 2024-11-20T21:29:08,010 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0b16ab3f13b7:33007 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0b16ab3f13b7:33007 2024-11-20T21:29:08,010 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0b16ab3f13b7:33007 2024-11-20T21:29:08,010 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,33007,1732138122755, seqNum=12 from cache 2024-11-20T21:29:08,227 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e., hostname=0b16ab3f13b7,45827,1732138122912, seqNum=18] 2024-11-20T21:29:08,247 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 411), OpenFileDescriptor=1042 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=688 (was 617) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3890 (was 3627) - AvailableMemoryMB LEAK? - 2024-11-20T21:29:08,249 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-20T21:29:08,261 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1042, MaxFileDescriptor=1048576, SystemLoadAverage=688, ProcessCount=11, AvailableMemoryMB=3889 2024-11-20T21:29:08,261 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-20T21:29:08,279 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:08,281 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:08,282 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:29:08,285 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72375771, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-72375771, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:08,305 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72375771/hregion-72375771.1732138148285, exclude list is [], retry=0 2024-11-20T21:29:08,307 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:08,308 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:08,309 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:08,345 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72375771/hregion-72375771.1732138148285 2024-11-20T21:29:08,345 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:08,345 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 75ee66fee352c779f236995b664bf408, NAME => 'testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:29:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741898_1076 (size=67) 2024-11-20T21:29:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741898_1076 (size=67) 2024-11-20T21:29:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741898_1076 (size=67) 2024-11-20T21:29:08,368 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:08,370 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,371 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName a 2024-11-20T21:29:08,371 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,372 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,372 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,373 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName b 2024-11-20T21:29:08,373 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,374 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,374 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,375 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName c 2024-11-20T21:29:08,375 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,375 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,376 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,376 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,377 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,378 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,378 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,378 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:08,379 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,381 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:29:08,381 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75ee66fee352c779f236995b664bf408; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61094176, jitterRate=-0.08962583541870117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:08,382 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75ee66fee352c779f236995b664bf408: Writing region info on filesystem at 1732138148368Initializing all the Stores at 1732138148369 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148369Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148370 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148370Cleaning up temporary data from old regions at 1732138148378 (+8 ms)Region opened successfully at 1732138148382 (+4 ms) 2024-11-20T21:29:08,382 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 75ee66fee352c779f236995b664bf408, disabling compactions & flushes 2024-11-20T21:29:08,382 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,382 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,382 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. after waiting 0 ms 2024-11-20T21:29:08,382 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,383 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,383 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 75ee66fee352c779f236995b664bf408: Waiting for close lock at 1732138148382Disabling compacts and flushes for region at 1732138148382Disabling writes for close at 1732138148382Writing region close event to WAL at 1732138148383 (+1 ms)Closed at 1732138148383 2024-11-20T21:29:08,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741897_1075 (size=175) 2024-11-20T21:29:08,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741897_1075 (size=175) 2024-11-20T21:29:08,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741897_1075 (size=175) 2024-11-20T21:29:08,388 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:08,388 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72375771:(num 1732138148285) 2024-11-20T21:29:08,388 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:08,389 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:08,402 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390, exclude list is [], retry=0 2024-11-20T21:29:08,405 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:08,405 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:08,405 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:08,433 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390 2024-11-20T21:29:08,433 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:08,433 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 75ee66fee352c779f236995b664bf408, NAME => 'testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:08,433 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:08,433 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,433 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,435 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,436 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName a 2024-11-20T21:29:08,436 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,437 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,437 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,438 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName b 2024-11-20T21:29:08,438 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,438 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,438 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,439 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName c 2024-11-20T21:29:08,439 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,440 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,440 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,440 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,441 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,443 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,443 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,443 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:08,444 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,445 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75ee66fee352c779f236995b664bf408; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67484303, jitterRate=0.00559447705745697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:08,446 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75ee66fee352c779f236995b664bf408: Writing region info on filesystem at 1732138148433Initializing all the Stores at 1732138148434 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148434Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148435 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148435Cleaning up temporary data from old regions at 1732138148443 (+8 ms)Region opened successfully at 1732138148446 (+3 ms) 2024-11-20T21:29:08,469 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 75ee66fee352c779f236995b664bf408 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T21:29:08,489 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/a/68387edcd6494387a200e176c4c5254c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732138148446/Put/seqid=0 2024-11-20T21:29:08,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741900_1078 (size=5958) 2024-11-20T21:29:08,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741900_1078 (size=5958) 2024-11-20T21:29:08,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741900_1078 (size=5958) 2024-11-20T21:29:08,499 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/a/68387edcd6494387a200e176c4c5254c 2024-11-20T21:29:08,521 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/b/5b86dcdc101b4d00a51134891c2348ec is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732138148454/Put/seqid=0 2024-11-20T21:29:08,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741901_1079 (size=5958) 2024-11-20T21:29:08,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741901_1079 (size=5958) 2024-11-20T21:29:08,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741901_1079 (size=5958) 2024-11-20T21:29:08,531 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/b/5b86dcdc101b4d00a51134891c2348ec 2024-11-20T21:29:08,551 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/c/0bfbef3e5fde42fd85f2a21f0f5bc406 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732138148461/Put/seqid=0 2024-11-20T21:29:08,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741902_1080 (size=5958) 2024-11-20T21:29:08,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741902_1080 (size=5958) 2024-11-20T21:29:08,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741902_1080 (size=5958) 2024-11-20T21:29:08,567 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/c/0bfbef3e5fde42fd85f2a21f0f5bc406 2024-11-20T21:29:08,576 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/a/68387edcd6494387a200e176c4c5254c as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/a/68387edcd6494387a200e176c4c5254c 2024-11-20T21:29:08,582 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/a/68387edcd6494387a200e176c4c5254c, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T21:29:08,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/b/5b86dcdc101b4d00a51134891c2348ec as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/b/5b86dcdc101b4d00a51134891c2348ec 2024-11-20T21:29:08,589 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/b/5b86dcdc101b4d00a51134891c2348ec, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T21:29:08,589 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/c/0bfbef3e5fde42fd85f2a21f0f5bc406 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/c/0bfbef3e5fde42fd85f2a21f0f5bc406 2024-11-20T21:29:08,597 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/c/0bfbef3e5fde42fd85f2a21f0f5bc406, entries=10, sequenceid=33, filesize=5.8 K 2024-11-20T21:29:08,598 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 75ee66fee352c779f236995b664bf408 in 129ms, sequenceid=33, compaction requested=false 2024-11-20T21:29:08,598 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 75ee66fee352c779f236995b664bf408: 2024-11-20T21:29:08,599 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 75ee66fee352c779f236995b664bf408, disabling compactions & flushes 2024-11-20T21:29:08,599 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,599 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,599 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. after waiting 0 ms 2024-11-20T21:29:08,599 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,600 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. 2024-11-20T21:29:08,600 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 75ee66fee352c779f236995b664bf408: Waiting for close lock at 1732138148599Disabling compacts and flushes for region at 1732138148599Disabling writes for close at 1732138148599Writing region close event to WAL at 1732138148600 (+1 ms)Closed at 1732138148600 2024-11-20T21:29:08,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741899_1077 (size=6311) 2024-11-20T21:29:08,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741899_1077 (size=6311) 2024-11-20T21:29:08,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741899_1077 (size=6311) 2024-11-20T21:29:08,614 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/b/5b86dcdc101b4d00a51134891c2348ec to hdfs://localhost:44823/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/b/5b86dcdc101b4d00a51134891c2348ec 2024-11-20T21:29:08,638 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390, size=6.2 K (6311bytes) 2024-11-20T21:29:08,638 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390 2024-11-20T21:29:08,640 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390 after 2ms 2024-11-20T21:29:08,688 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:08,688 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390 took 50ms 2024-11-20T21:29:08,699 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390 so closing down 2024-11-20T21:29:08,699 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:08,702 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732138148390.temp 2024-11-20T21:29:08,736 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000003-wal.1732138148390.temp 2024-11-20T21:29:08,736 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:08,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741903_1081 (size=5575) 2024-11-20T21:29:08,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741903_1081 (size=5575) 2024-11-20T21:29:08,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741903_1081 (size=5575) 2024-11-20T21:29:08,746 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000003-wal.1732138148390.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:08,748 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000003-wal.1732138148390.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000032 2024-11-20T21:29:08,748 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-20T21:29:08,749 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390, journal: Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390, size=6.2 K (6311bytes) at 1732138148638Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390 so closing down at 1732138148699 (+61 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000003-wal.1732138148390.temp at 1732138148736 (+37 ms)3 split writer threads finished at 1732138148736Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000003-wal.1732138148390.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732138148746 (+10 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000003-wal.1732138148390.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000032 at 1732138148748 (+2 ms)Processed 32 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732138148748 2024-11-20T21:29:08,751 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148390 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138148390 2024-11-20T21:29:08,752 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000032 2024-11-20T21:29:08,752 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:08,754 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:08,774 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148754, exclude list is [], retry=0 2024-11-20T21:29:08,777 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:08,778 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:08,778 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:08,810 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148754 2024-11-20T21:29:08,810 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:08,811 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 75ee66fee352c779f236995b664bf408, NAME => 'testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:08,811 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:08,811 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,811 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,813 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,814 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName a 2024-11-20T21:29:08,814 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,820 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/a/68387edcd6494387a200e176c4c5254c 2024-11-20T21:29:08,820 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,821 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,823 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName b 2024-11-20T21:29:08,824 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,824 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,824 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,826 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75ee66fee352c779f236995b664bf408 columnFamilyName c 2024-11-20T21:29:08,826 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:08,836 DEBUG [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/c/0bfbef3e5fde42fd85f2a21f0f5bc406 2024-11-20T21:29:08,836 INFO [StoreOpener-75ee66fee352c779f236995b664bf408-1 {}] regionserver.HStore(327): Store=75ee66fee352c779f236995b664bf408/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:08,836 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,846 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,847 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000032 2024-11-20T21:29:08,883 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:08,885 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000032 2024-11-20T21:29:08,886 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 75ee66fee352c779f236995b664bf408 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T21:29:08,908 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/b/9cf9a9a2e6514affb3241ee93fec1793 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732138148454/Put/seqid=0 2024-11-20T21:29:08,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741905_1083 (size=5958) 2024-11-20T21:29:08,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741905_1083 (size=5958) 2024-11-20T21:29:08,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741905_1083 (size=5958) 2024-11-20T21:29:08,949 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/b/9cf9a9a2e6514affb3241ee93fec1793 2024-11-20T21:29:08,966 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/.tmp/b/9cf9a9a2e6514affb3241ee93fec1793 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/b/9cf9a9a2e6514affb3241ee93fec1793 2024-11-20T21:29:08,982 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/b/9cf9a9a2e6514affb3241ee93fec1793, entries=10, sequenceid=32, filesize=5.8 K 2024-11-20T21:29:08,982 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 75ee66fee352c779f236995b664bf408 in 97ms, sequenceid=32, compaction requested=false; wal=null 2024-11-20T21:29:08,983 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/0000000000000000032 2024-11-20T21:29:08,988 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,988 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:08,990 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:08,995 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 75ee66fee352c779f236995b664bf408 2024-11-20T21:29:09,013 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/75ee66fee352c779f236995b664bf408/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-20T21:29:09,015 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 75ee66fee352c779f236995b664bf408; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60933456, jitterRate=-0.09202075004577637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:09,016 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 75ee66fee352c779f236995b664bf408: Writing region info on filesystem at 1732138148811Initializing all the Stores at 1732138148812 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148812Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148813 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138148813Obtaining lock to block concurrent updates at 1732138148886 (+73 ms)Preparing flush snapshotting stores in 75ee66fee352c779f236995b664bf408 at 1732138148886Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732138148886Flushing stores of testReplayEditsWrittenViaHRegion,,1732138148280.75ee66fee352c779f236995b664bf408. at 1732138148886Flushing 75ee66fee352c779f236995b664bf408/b: creating writer at 1732138148886Flushing 75ee66fee352c779f236995b664bf408/b: appending metadata at 1732138148908 (+22 ms)Flushing 75ee66fee352c779f236995b664bf408/b: closing flushed file at 1732138148908Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@402c89e7: reopening flushed file at 1732138148964 (+56 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 75ee66fee352c779f236995b664bf408 in 97ms, sequenceid=32, compaction requested=false; wal=null at 1732138148982 (+18 ms)Cleaning up temporary data from old regions at 1732138148988 (+6 ms)Region opened successfully at 1732138149016 (+28 ms) 2024-11-20T21:29:09,047 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409 (was 405) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:33064 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:58754 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:60554 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:47214 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:57728 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:40294 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1110 (was 1042) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=688 (was 688), ProcessCount=11 (was 11), AvailableMemoryMB=3890 (was 3889) - AvailableMemoryMB LEAK? - 2024-11-20T21:29:09,047 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1110 is superior to 1024 2024-11-20T21:29:09,062 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=409, OpenFileDescriptor=1110, MaxFileDescriptor=1048576, SystemLoadAverage=688, ProcessCount=11, AvailableMemoryMB=3885 2024-11-20T21:29:09,062 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1110 is superior to 1024 2024-11-20T21:29:09,086 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:09,091 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:09,091 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:29:09,104 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-61777004, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-61777004, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:09,126 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-61777004/hregion-61777004.1732138149105, exclude list is [], retry=0 2024-11-20T21:29:09,134 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:09,135 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:09,137 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:09,170 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-61777004/hregion-61777004.1732138149105 2024-11-20T21:29:09,170 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:29:09,170 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fa8ab0f521b3459d2c5aface5631fd0a, NAME => 'testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:29:09,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741907_1085 (size=68) 2024-11-20T21:29:09,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741907_1085 (size=68) 2024-11-20T21:29:09,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741907_1085 (size=68) 2024-11-20T21:29:09,179 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:09,181 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,182 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName a 2024-11-20T21:29:09,182 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:09,182 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:09,183 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,184 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName b 2024-11-20T21:29:09,184 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:09,185 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:09,185 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,186 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName c 2024-11-20T21:29:09,186 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:09,186 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:09,186 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,187 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,187 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,188 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,188 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,189 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:09,189 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:29:09,190 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,193 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:29:09,193 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fa8ab0f521b3459d2c5aface5631fd0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72637177, jitterRate=0.08237828314304352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:09,194 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fa8ab0f521b3459d2c5aface5631fd0a: Writing region info on filesystem at 1732138149179Initializing all the Stores at 1732138149180 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138149180Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138149180Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138149180Cleaning up temporary data from old regions at 1732138149188 (+8 ms)Region opened successfully at 1732138149194 (+6 ms) 2024-11-20T21:29:09,194 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fa8ab0f521b3459d2c5aface5631fd0a, disabling compactions & flushes 2024-11-20T21:29:09,194 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:09,194 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:09,194 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. after waiting 0 ms 2024-11-20T21:29:09,194 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:09,196 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:09,196 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fa8ab0f521b3459d2c5aface5631fd0a: Waiting for close lock at 1732138149194Disabling compacts and flushes for region at 1732138149194Disabling writes for close at 1732138149194Writing region close event to WAL at 1732138149196 (+2 ms)Closed at 1732138149196 2024-11-20T21:29:09,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741906_1084 (size=175) 2024-11-20T21:29:09,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741906_1084 (size=175) 2024-11-20T21:29:09,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741906_1084 (size=175) 2024-11-20T21:29:09,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:09,205 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-61777004:(num 1732138149105) 2024-11-20T21:29:09,205 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:09,206 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:09,220 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207, exclude list is [], retry=0 2024-11-20T21:29:09,222 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:09,223 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:09,223 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:09,249 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 2024-11-20T21:29:09,249 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729)] 2024-11-20T21:29:09,299 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fa8ab0f521b3459d2c5aface5631fd0a, NAME => 'testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:09,301 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,302 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:09,302 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,302 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,304 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,304 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName a 2024-11-20T21:29:09,305 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:09,305 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:09,305 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,306 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName b 2024-11-20T21:29:09,306 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:09,306 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:09,307 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,307 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName c 2024-11-20T21:29:09,307 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:09,308 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:09,308 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,308 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,309 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,310 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,310 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,311 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:09,312 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,313 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fa8ab0f521b3459d2c5aface5631fd0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58976945, jitterRate=-0.12117503583431244}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:09,313 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:09,314 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fa8ab0f521b3459d2c5aface5631fd0a: Running coprocessor pre-open hook at 1732138149302Writing region info on filesystem at 1732138149302Initializing all the Stores at 1732138149303 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138149303Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138149303Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138149303Cleaning up temporary data from old regions at 1732138149310 (+7 ms)Running coprocessor post-open hooks at 1732138149313 (+3 ms)Region opened successfully at 1732138149314 (+1 ms) 2024-11-20T21:29:09,327 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fa8ab0f521b3459d2c5aface5631fd0a 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-20T21:29:09,328 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:10,329 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:10,631 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T21:29:11,329 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:12,057 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-20T21:29:12,057 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-20T21:29:12,059 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-20T21:29:12,059 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-20T21:29:12,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-20T21:29:12,060 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-20T21:29:12,330 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:13,331 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:14,332 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:14,804 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:29:15,333 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:16,333 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:17,334 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:18,335 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for fa8ab0f521b3459d2c5aface5631fd0a/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:18,336 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fa8ab0f521b3459d2c5aface5631fd0a: 2024-11-20T21:29:18,336 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:18,349 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fa8ab0f521b3459d2c5aface5631fd0a: 2024-11-20T21:29:18,349 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-20T21:29:18,349 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fa8ab0f521b3459d2c5aface5631fd0a, disabling compactions & flushes 2024-11-20T21:29:18,349 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:18,349 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:18,349 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. after waiting 0 ms 2024-11-20T21:29:18,349 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:18,350 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:18,350 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. 2024-11-20T21:29:18,350 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fa8ab0f521b3459d2c5aface5631fd0a: Waiting for close lock at 1732138158349Running coprocessor pre-close hooks at 1732138158349Disabling compacts and flushes for region at 1732138158349Disabling writes for close at 1732138158349Writing region close event to WAL at 1732138158350 (+1 ms)Running coprocessor post-close hooks at 1732138158350Closed at 1732138158350 2024-11-20T21:29:18,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741908_1086 (size=4236) 2024-11-20T21:29:18,354 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 not finished, retry = 0 2024-11-20T21:29:18,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741908_1086 (size=4236) 2024-11-20T21:29:18,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741908_1086 (size=4236) 2024-11-20T21:29:18,475 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207, size=4.1 K (4236bytes) 2024-11-20T21:29:18,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 2024-11-20T21:29:18,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 after 0ms 2024-11-20T21:29:18,510 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:18,510 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 took 35ms 2024-11-20T21:29:18,545 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 so closing down 2024-11-20T21:29:18,545 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:18,546 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732138149207.temp 2024-11-20T21:29:18,579 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000004-wal.1732138149207.temp 2024-11-20T21:29:18,580 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:18,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741909_1087 (size=3245) 2024-11-20T21:29:18,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741909_1087 (size=3245) 2024-11-20T21:29:18,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741909_1087 (size=3245) 2024-11-20T21:29:18,589 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000004-wal.1732138149207.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:18,591 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000004-wal.1732138149207.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000026 2024-11-20T21:29:18,591 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 48 ms; skipped=3; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-20T21:29:18,591 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207, journal: Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207, size=4.1 K (4236bytes) at 1732138158475Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 so closing down at 1732138158545 (+70 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000004-wal.1732138149207.temp at 1732138158579 (+34 ms)3 split writer threads finished at 1732138158580 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000004-wal.1732138149207.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732138158589 (+9 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000004-wal.1732138149207.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000026 at 1732138158591 (+2 ms)Processed 23 edits across 1 Regions in 48 ms; skipped=3; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732138158591 2024-11-20T21:29:18,593 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138149207 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138149207 2024-11-20T21:29:18,594 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000026 2024-11-20T21:29:18,594 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:18,596 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:18,611 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138158596, exclude list is [], retry=0 2024-11-20T21:29:18,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:18,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:18,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:18,641 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138158596 2024-11-20T21:29:18,641 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:29:18,642 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fa8ab0f521b3459d2c5aface5631fd0a, NAME => 'testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:18,642 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,642 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:18,642 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,643 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,644 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,645 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName a 2024-11-20T21:29:18,645 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:18,646 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:18,646 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,647 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName b 2024-11-20T21:29:18,647 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:18,648 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:18,648 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,648 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8ab0f521b3459d2c5aface5631fd0a columnFamilyName c 2024-11-20T21:29:18,648 DEBUG [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:18,649 INFO [StoreOpener-fa8ab0f521b3459d2c5aface5631fd0a-1 {}] regionserver.HStore(327): Store=fa8ab0f521b3459d2c5aface5631fd0a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:18,649 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,650 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,651 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000026 2024-11-20T21:29:18,687 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:18,688 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000026 2024-11-20T21:29:18,689 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fa8ab0f521b3459d2c5aface5631fd0a 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-20T21:29:18,704 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/a/e11e0aa587804bc394aff5135f0f5475 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732138158340/Put/seqid=0 2024-11-20T21:29:18,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741911_1089 (size=5523) 2024-11-20T21:29:18,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741911_1089 (size=5523) 2024-11-20T21:29:18,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741911_1089 (size=5523) 2024-11-20T21:29:18,720 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/a/e11e0aa587804bc394aff5135f0f5475 2024-11-20T21:29:18,753 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/b/2bb3d55849b84fdca7be5eb81e65a13b is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732138158336/Put/seqid=0 2024-11-20T21:29:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741912_1090 (size=5524) 2024-11-20T21:29:18,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741912_1090 (size=5524) 2024-11-20T21:29:18,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741912_1090 (size=5524) 2024-11-20T21:29:18,763 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/b/2bb3d55849b84fdca7be5eb81e65a13b 2024-11-20T21:29:18,790 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/c/8059f50daeb64708a536f5cf796b9f32 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732138158338/Put/seqid=0 2024-11-20T21:29:18,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741913_1091 (size=5457) 2024-11-20T21:29:18,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741913_1091 (size=5457) 2024-11-20T21:29:18,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741913_1091 (size=5457) 2024-11-20T21:29:18,801 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/c/8059f50daeb64708a536f5cf796b9f32 2024-11-20T21:29:18,808 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/a/e11e0aa587804bc394aff5135f0f5475 as hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/a/e11e0aa587804bc394aff5135f0f5475 2024-11-20T21:29:18,813 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/a/e11e0aa587804bc394aff5135f0f5475, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T21:29:18,814 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/b/2bb3d55849b84fdca7be5eb81e65a13b as hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/b/2bb3d55849b84fdca7be5eb81e65a13b 2024-11-20T21:29:18,819 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/b/2bb3d55849b84fdca7be5eb81e65a13b, entries=7, sequenceid=26, filesize=5.4 K 2024-11-20T21:29:18,821 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/.tmp/c/8059f50daeb64708a536f5cf796b9f32 as hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/c/8059f50daeb64708a536f5cf796b9f32 2024-11-20T21:29:18,827 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/c/8059f50daeb64708a536f5cf796b9f32, entries=6, sequenceid=26, filesize=5.3 K 2024-11-20T21:29:18,827 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for fa8ab0f521b3459d2c5aface5631fd0a in 138ms, sequenceid=26, compaction requested=false; wal=null 2024-11-20T21:29:18,828 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/0000000000000000026 2024-11-20T21:29:18,829 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,829 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,830 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:18,831 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,834 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsAfterAbortingFlush/fa8ab0f521b3459d2c5aface5631fd0a/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-20T21:29:18,835 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fa8ab0f521b3459d2c5aface5631fd0a; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70872068, jitterRate=0.056076109409332275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:18,835 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa8ab0f521b3459d2c5aface5631fd0a 2024-11-20T21:29:18,835 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fa8ab0f521b3459d2c5aface5631fd0a: Running coprocessor pre-open hook at 1732138158643Writing region info on filesystem at 1732138158643Initializing all the Stores at 1732138158644 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138158644Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138158644Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138158644Obtaining lock to block concurrent updates at 1732138158689 (+45 ms)Preparing flush snapshotting stores in fa8ab0f521b3459d2c5aface5631fd0a at 1732138158689Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732138158689Flushing stores of testReplayEditsAfterAbortingFlush,,1732138149088.fa8ab0f521b3459d2c5aface5631fd0a. at 1732138158689Flushing fa8ab0f521b3459d2c5aface5631fd0a/a: creating writer at 1732138158689Flushing fa8ab0f521b3459d2c5aface5631fd0a/a: appending metadata at 1732138158704 (+15 ms)Flushing fa8ab0f521b3459d2c5aface5631fd0a/a: closing flushed file at 1732138158704Flushing fa8ab0f521b3459d2c5aface5631fd0a/b: creating writer at 1732138158728 (+24 ms)Flushing fa8ab0f521b3459d2c5aface5631fd0a/b: appending metadata at 1732138158753 (+25 ms)Flushing fa8ab0f521b3459d2c5aface5631fd0a/b: closing flushed file at 1732138158753Flushing fa8ab0f521b3459d2c5aface5631fd0a/c: creating writer at 1732138158769 (+16 ms)Flushing fa8ab0f521b3459d2c5aface5631fd0a/c: appending metadata at 1732138158789 (+20 ms)Flushing fa8ab0f521b3459d2c5aface5631fd0a/c: closing flushed file at 1732138158790 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24e7ea1: reopening flushed file at 1732138158807 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61776cd5: reopening flushed file at 1732138158813 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d8555f5: reopening flushed file at 1732138158819 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for fa8ab0f521b3459d2c5aface5631fd0a in 138ms, sequenceid=26, compaction requested=false; wal=null at 1732138158827 (+8 ms)Cleaning up temporary data from old regions at 1732138158829 (+2 ms)Running coprocessor post-open hooks at 1732138158835 (+6 ms)Region opened successfully at 1732138158835 2024-11-20T21:29:18,858 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=416 (was 409) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:48174 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:41840 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:48160 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:41852 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:45554 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:45534 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1110) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=635 (was 688), ProcessCount=11 (was 11), AvailableMemoryMB=3177 (was 3885) 2024-11-20T21:29:18,858 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-20T21:29:18,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=416, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=635, ProcessCount=11, AvailableMemoryMB=3176 2024-11-20T21:29:18,871 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-20T21:29:18,886 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:18,887 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:18,888 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:29:18,890 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41939480, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-41939480, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:18,903 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41939480/hregion-41939480.1732138158891, exclude list is [], retry=0 2024-11-20T21:29:18,905 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:18,905 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:18,906 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:18,933 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41939480/hregion-41939480.1732138158891 2024-11-20T21:29:18,934 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:18,934 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 49aa7c86c4c94fda17fe950f3a73961a, NAME => 'testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:29:18,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741915_1093 (size=61) 2024-11-20T21:29:18,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741915_1093 (size=61) 2024-11-20T21:29:18,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741915_1093 (size=61) 2024-11-20T21:29:18,942 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:18,944 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:18,945 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49aa7c86c4c94fda17fe950f3a73961a columnFamilyName a 2024-11-20T21:29:18,945 DEBUG [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:18,946 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(327): Store=49aa7c86c4c94fda17fe950f3a73961a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:18,946 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:18,947 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:18,947 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:18,947 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:18,947 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:18,950 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:18,952 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:29:18,953 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 49aa7c86c4c94fda17fe950f3a73961a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66481380, jitterRate=-0.009350240230560303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:29:18,953 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 49aa7c86c4c94fda17fe950f3a73961a: Writing region info on filesystem at 1732138158943Initializing all the Stores at 1732138158943Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138158943Cleaning up temporary data from old regions at 1732138158947 (+4 ms)Region opened successfully at 1732138158953 (+6 ms) 2024-11-20T21:29:18,954 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 49aa7c86c4c94fda17fe950f3a73961a, disabling compactions & flushes 2024-11-20T21:29:18,954 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:18,954 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:18,954 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. after waiting 0 ms 2024-11-20T21:29:18,954 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:18,954 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:18,954 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 49aa7c86c4c94fda17fe950f3a73961a: Waiting for close lock at 1732138158953Disabling compacts and flushes for region at 1732138158953Disabling writes for close at 1732138158954 (+1 ms)Writing region close event to WAL at 1732138158954Closed at 1732138158954 2024-11-20T21:29:18,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741914_1092 (size=175) 2024-11-20T21:29:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741914_1092 (size=175) 2024-11-20T21:29:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741914_1092 (size=175) 2024-11-20T21:29:18,970 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:18,970 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41939480:(num 1732138158891) 2024-11-20T21:29:18,971 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:18,973 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:18,989 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973, exclude list is [], retry=0 2024-11-20T21:29:18,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:18,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:18,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:19,019 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 2024-11-20T21:29:19,019 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:19,019 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 49aa7c86c4c94fda17fe950f3a73961a, NAME => 'testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:19,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:19,019 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,019 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,021 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,022 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49aa7c86c4c94fda17fe950f3a73961a columnFamilyName a 2024-11-20T21:29:19,022 DEBUG [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,022 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(327): Store=49aa7c86c4c94fda17fe950f3a73961a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,022 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,024 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,025 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,025 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,026 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 49aa7c86c4c94fda17fe950f3a73961a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65514142, jitterRate=-0.023763209581375122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:29:19,028 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 49aa7c86c4c94fda17fe950f3a73961a: Writing region info on filesystem at 1732138159020Initializing all the Stores at 1732138159020Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159020Cleaning up temporary data from old regions at 1732138159025 (+5 ms)Region opened successfully at 1732138159028 (+3 ms) 2024-11-20T21:29:19,036 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 49aa7c86c4c94fda17fe950f3a73961a, disabling compactions & flushes 2024-11-20T21:29:19,036 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:19,036 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:19,036 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. after waiting 0 ms 2024-11-20T21:29:19,036 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:19,036 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:19,036 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. 2024-11-20T21:29:19,036 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 49aa7c86c4c94fda17fe950f3a73961a: Waiting for close lock at 1732138159036Disabling compacts and flushes for region at 1732138159036Disabling writes for close at 1732138159036Writing region close event to WAL at 1732138159036Closed at 1732138159036 2024-11-20T21:29:19,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741916_1094 (size=1795) 2024-11-20T21:29:19,039 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 not finished, retry = 0 2024-11-20T21:29:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741916_1094 (size=1795) 2024-11-20T21:29:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741916_1094 (size=1795) 2024-11-20T21:29:19,156 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973, size=1.8 K (1795bytes) 2024-11-20T21:29:19,156 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 2024-11-20T21:29:19,157 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 after 1ms 2024-11-20T21:29:19,182 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:19,182 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 took 26ms 2024-11-20T21:29:19,184 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 so closing down 2024-11-20T21:29:19,184 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:19,186 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732138158973.temp 2024-11-20T21:29:19,210 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000003-wal.1732138158973.temp 2024-11-20T21:29:19,210 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:19,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741917_1095 (size=1795) 2024-11-20T21:29:19,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741917_1095 (size=1795) 2024-11-20T21:29:19,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741917_1095 (size=1795) 2024-11-20T21:29:19,217 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000003-wal.1732138158973.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:19,218 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000003-wal.1732138158973.temp to hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000012 2024-11-20T21:29:19,218 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-20T21:29:19,219 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973, journal: Splitting hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973, size=1.8 K (1795bytes) at 1732138159156Finishing writing output for hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 so closing down at 1732138159184 (+28 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000003-wal.1732138158973.temp at 1732138159210 (+26 ms)3 split writer threads finished at 1732138159210Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000003-wal.1732138158973.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732138159217 (+7 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000003-wal.1732138158973.temp to hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000012 at 1732138159218 (+1 ms)Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732138159218 2024-11-20T21:29:19,220 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138158973 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138158973 2024-11-20T21:29:19,221 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000012 2024-11-20T21:29:19,251 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:19,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:19,614 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:19,627 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138159614, exclude list is [], retry=0 2024-11-20T21:29:19,629 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:19,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:19,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:19,658 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138159614 2024-11-20T21:29:19,658 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:34363:34363)] 2024-11-20T21:29:19,658 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 49aa7c86c4c94fda17fe950f3a73961a, NAME => 'testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:19,658 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:19,659 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,659 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,662 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,663 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49aa7c86c4c94fda17fe950f3a73961a columnFamilyName a 2024-11-20T21:29:19,663 DEBUG [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,664 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(327): Store=49aa7c86c4c94fda17fe950f3a73961a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,664 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,665 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,666 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,667 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000012 2024-11-20T21:29:19,692 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:19,693 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000012 2024-11-20T21:29:19,693 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 49aa7c86c4c94fda17fe950f3a73961a 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-20T21:29:19,710 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/.tmp/a/15e204f497a0417091ec05d6acb0785b is 79, key is testDatalossWhenInputError/a:x0/1732138159028/Put/seqid=0 2024-11-20T21:29:19,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741919_1097 (size=5808) 2024-11-20T21:29:19,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741919_1097 (size=5808) 2024-11-20T21:29:19,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741919_1097 (size=5808) 2024-11-20T21:29:19,717 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/.tmp/a/15e204f497a0417091ec05d6acb0785b 2024-11-20T21:29:19,725 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/.tmp/a/15e204f497a0417091ec05d6acb0785b as hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/a/15e204f497a0417091ec05d6acb0785b 2024-11-20T21:29:19,733 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/a/15e204f497a0417091ec05d6acb0785b, entries=10, sequenceid=12, filesize=5.7 K 2024-11-20T21:29:19,733 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 49aa7c86c4c94fda17fe950f3a73961a in 40ms, sequenceid=12, compaction requested=false; wal=null 2024-11-20T21:29:19,733 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/0000000000000000012 2024-11-20T21:29:19,734 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,734 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,737 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,739 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-20T21:29:19,740 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 49aa7c86c4c94fda17fe950f3a73961a; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64674699, jitterRate=-0.03627188503742218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:29:19,740 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 49aa7c86c4c94fda17fe950f3a73961a: Writing region info on filesystem at 1732138159659Initializing all the Stores at 1732138159661 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159661Obtaining lock to block concurrent updates at 1732138159694 (+33 ms)Preparing flush snapshotting stores in 49aa7c86c4c94fda17fe950f3a73961a at 1732138159694Finished memstore snapshotting testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732138159694Flushing stores of testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a. at 1732138159694Flushing 49aa7c86c4c94fda17fe950f3a73961a/a: creating writer at 1732138159694Flushing 49aa7c86c4c94fda17fe950f3a73961a/a: appending metadata at 1732138159709 (+15 ms)Flushing 49aa7c86c4c94fda17fe950f3a73961a/a: closing flushed file at 1732138159709Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@296d6358: reopening flushed file at 1732138159724 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 49aa7c86c4c94fda17fe950f3a73961a in 40ms, sequenceid=12, compaction requested=false; wal=null at 1732138159733 (+9 ms)Cleaning up temporary data from old regions at 1732138159734 (+1 ms)Region opened successfully at 1732138159740 (+6 ms) 2024-11-20T21:29:19,743 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 49aa7c86c4c94fda17fe950f3a73961a, NAME => 'testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:19,743 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732138158886.49aa7c86c4c94fda17fe950f3a73961a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:19,743 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,743 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,745 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,746 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49aa7c86c4c94fda17fe950f3a73961a columnFamilyName a 2024-11-20T21:29:19,746 DEBUG [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,752 DEBUG [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/a/15e204f497a0417091ec05d6acb0785b 2024-11-20T21:29:19,752 INFO [StoreOpener-49aa7c86c4c94fda17fe950f3a73961a-1 {}] regionserver.HStore(327): Store=49aa7c86c4c94fda17fe950f3a73961a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,752 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,753 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,755 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,755 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,757 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 49aa7c86c4c94fda17fe950f3a73961a 2024-11-20T21:29:19,760 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testDatalossWhenInputError/49aa7c86c4c94fda17fe950f3a73961a/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-20T21:29:19,761 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 49aa7c86c4c94fda17fe950f3a73961a; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65093968, jitterRate=-0.030024290084838867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T21:29:19,761 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 49aa7c86c4c94fda17fe950f3a73961a: Writing region info on filesystem at 1732138159743Initializing all the Stores at 1732138159744 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159744Cleaning up temporary data from old regions at 1732138159755 (+11 ms)Region opened successfully at 1732138159761 (+6 ms) 2024-11-20T21:29:19,783 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=426 (was 416) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:48174 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:41914 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:48240 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:41840 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-422807794-172.17.0.3-1732138113638:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:45554 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:45602 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=635 (was 635), ProcessCount=11 (was 11), AvailableMemoryMB=3182 (was 3176) - AvailableMemoryMB LEAK? - 2024-11-20T21:29:19,783 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-20T21:29:19,796 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=426, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=635, ProcessCount=11, AvailableMemoryMB=3179 2024-11-20T21:29:19,796 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-20T21:29:19,812 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:19,814 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:19,815 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:29:19,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16186259, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-16186259, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:19,830 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16186259/hregion-16186259.1732138159817, exclude list is [], retry=0 2024-11-20T21:29:19,832 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:19,832 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:19,833 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:19,860 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16186259/hregion-16186259.1732138159817 2024-11-20T21:29:19,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729)] 2024-11-20T21:29:19,860 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => db3cca17488e26661a37c81a08cc0d57, NAME => 'testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:29:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741921_1099 (size=63) 2024-11-20T21:29:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741921_1099 (size=63) 2024-11-20T21:29:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741921_1099 (size=63) 2024-11-20T21:29:19,873 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:19,875 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,876 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName a 2024-11-20T21:29:19,876 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,877 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,877 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,878 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName b 2024-11-20T21:29:19,878 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,879 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,879 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,881 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName c 2024-11-20T21:29:19,881 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,882 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,882 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,883 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,883 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,887 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,887 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,888 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:19,890 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,892 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:29:19,893 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db3cca17488e26661a37c81a08cc0d57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73647343, jitterRate=0.09743092954158783}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:19,893 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db3cca17488e26661a37c81a08cc0d57: Writing region info on filesystem at 1732138159873Initializing all the Stores at 1732138159874 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159874Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159874Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159874Cleaning up temporary data from old regions at 1732138159887 (+13 ms)Region opened successfully at 1732138159893 (+6 ms) 2024-11-20T21:29:19,893 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing db3cca17488e26661a37c81a08cc0d57, disabling compactions & flushes 2024-11-20T21:29:19,893 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:19,893 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:19,893 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. after waiting 0 ms 2024-11-20T21:29:19,893 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:19,894 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:19,894 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for db3cca17488e26661a37c81a08cc0d57: Waiting for close lock at 1732138159893Disabling compacts and flushes for region at 1732138159893Disabling writes for close at 1732138159893Writing region close event to WAL at 1732138159894 (+1 ms)Closed at 1732138159894 2024-11-20T21:29:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741920_1098 (size=175) 2024-11-20T21:29:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741920_1098 (size=175) 2024-11-20T21:29:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741920_1098 (size=175) 2024-11-20T21:29:19,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:19,900 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16186259:(num 1732138159817) 2024-11-20T21:29:19,900 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:19,902 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:19,917 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902, exclude list is [], retry=0 2024-11-20T21:29:19,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:19,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:19,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:19,957 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 2024-11-20T21:29:19,957 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:19,957 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => db3cca17488e26661a37c81a08cc0d57, NAME => 'testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:19,958 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:19,958 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,958 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,960 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,961 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName a 2024-11-20T21:29:19,962 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,962 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,962 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,963 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName b 2024-11-20T21:29:19,963 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,964 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,964 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,965 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName c 2024-11-20T21:29:19,965 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:19,966 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:19,966 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,968 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,968 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,968 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,969 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:19,970 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:19,971 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db3cca17488e26661a37c81a08cc0d57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71656127, jitterRate=0.06775949895381927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:19,972 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db3cca17488e26661a37c81a08cc0d57: Writing region info on filesystem at 1732138159958Initializing all the Stores at 1732138159960 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159960Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159960Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138159960Cleaning up temporary data from old regions at 1732138159968 (+8 ms)Region opened successfully at 1732138159972 (+4 ms) 2024-11-20T21:29:19,977 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732138159976/Put/seqid=0 2024-11-20T21:29:19,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741923_1101 (size=4875) 2024-11-20T21:29:19,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741923_1101 (size=4875) 2024-11-20T21:29:19,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741923_1101 (size=4875) 2024-11-20T21:29:19,998 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732138159998/Put/seqid=0 2024-11-20T21:29:20,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741924_1102 (size=4875) 2024-11-20T21:29:20,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741924_1102 (size=4875) 2024-11-20T21:29:20,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741924_1102 (size=4875) 2024-11-20T21:29:20,013 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732138160013/Put/seqid=0 2024-11-20T21:29:20,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741925_1103 (size=4875) 2024-11-20T21:29:20,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741925_1103 (size=4875) 2024-11-20T21:29:20,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741925_1103 (size=4875) 2024-11-20T21:29:20,023 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in db3cca17488e26661a37c81a08cc0d57/a 2024-11-20T21:29:20,027 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-20T21:29:20,027 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T21:29:20,028 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in db3cca17488e26661a37c81a08cc0d57/a 2024-11-20T21:29:20,031 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-20T21:29:20,031 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T21:29:20,031 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in db3cca17488e26661a37c81a08cc0d57/a 2024-11-20T21:29:20,034 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-20T21:29:20,034 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-20T21:29:20,034 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing db3cca17488e26661a37c81a08cc0d57 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-20T21:29:20,051 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/.tmp/a/35b1a8d0b07447fba4cb7664e8fb1f4c is 55, key is testCompactedBulkLoadedFiles/a:a/1732138159972/Put/seqid=0 2024-11-20T21:29:20,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741926_1104 (size=5107) 2024-11-20T21:29:20,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741926_1104 (size=5107) 2024-11-20T21:29:20,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741926_1104 (size=5107) 2024-11-20T21:29:20,058 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/.tmp/a/35b1a8d0b07447fba4cb7664e8fb1f4c 2024-11-20T21:29:20,067 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/.tmp/a/35b1a8d0b07447fba4cb7664e8fb1f4c as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c 2024-11-20T21:29:20,073 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c, entries=1, sequenceid=4, filesize=5.0 K 2024-11-20T21:29:20,076 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for db3cca17488e26661a37c81a08cc0d57 in 42ms, sequenceid=4, compaction requested=false 2024-11-20T21:29:20,077 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for db3cca17488e26661a37c81a08cc0d57: 2024-11-20T21:29:20,079 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_ 2024-11-20T21:29:20,084 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_ 2024-11-20T21:29:20,085 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_ 2024-11-20T21:29:20,086 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile0 into db3cca17488e26661a37c81a08cc0d57/a as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_ - updating store file list. 2024-11-20T21:29:20,091 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T21:29:20,091 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_ into db3cca17488e26661a37c81a08cc0d57/a 2024-11-20T21:29:20,092 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile0 into db3cca17488e26661a37c81a08cc0d57/a (new location: hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_) 2024-11-20T21:29:20,093 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile1 into db3cca17488e26661a37c81a08cc0d57/a as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_ - updating store file list. 2024-11-20T21:29:20,098 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 65a972e7fb6e439bb32a74014c931b12_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T21:29:20,098 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_ into db3cca17488e26661a37c81a08cc0d57/a 2024-11-20T21:29:20,098 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile1 into db3cca17488e26661a37c81a08cc0d57/a (new location: hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_) 2024-11-20T21:29:20,099 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile2 into db3cca17488e26661a37c81a08cc0d57/a as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_ - updating store file list. 2024-11-20T21:29:20,104 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T21:29:20,105 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_ into db3cca17488e26661a37c81a08cc0d57/a 2024-11-20T21:29:20,105 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44823/hbase/testCompactedBulkLoadedFiles/hfile2 into db3cca17488e26661a37c81a08cc0d57/a (new location: hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_) 2024-11-20T21:29:20,113 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:29:20,113 DEBUG [Time-limited test {}] regionserver.HStore(1541): db3cca17488e26661a37c81a08cc0d57/a is initiating major compaction (all files) 2024-11-20T21:29:20,113 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of db3cca17488e26661a37c81a08cc0d57/a in testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:20,113 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_] into tmpdir=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/.tmp, totalSize=19.3 K 2024-11-20T21:29:20,114 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 35b1a8d0b07447fba4cb7664e8fb1f4c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732138159972 2024-11-20T21:29:20,114 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T21:29:20,114 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 65a972e7fb6e439bb32a74014c931b12_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T21:29:20,115 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-20T21:29:20,132 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/.tmp/a/fc641b5d9dbd4fde81a88e0d19554a44 is 55, key is testCompactedBulkLoadedFiles/a:a/1732138159972/Put/seqid=0 2024-11-20T21:29:20,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741927_1105 (size=6154) 2024-11-20T21:29:20,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741927_1105 (size=6154) 2024-11-20T21:29:20,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741927_1105 (size=6154) 2024-11-20T21:29:20,148 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/.tmp/a/fc641b5d9dbd4fde81a88e0d19554a44 as hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/fc641b5d9dbd4fde81a88e0d19554a44 2024-11-20T21:29:20,154 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in db3cca17488e26661a37c81a08cc0d57/a of db3cca17488e26661a37c81a08cc0d57 into fc641b5d9dbd4fde81a88e0d19554a44(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:29:20,154 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for db3cca17488e26661a37c81a08cc0d57: 2024-11-20T21:29:20,154 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T21:29:20,154 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-20T21:29:20,186 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902, size=0 (0bytes) 2024-11-20T21:29:20,186 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 might be still open, length is 0 2024-11-20T21:29:20,186 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 2024-11-20T21:29:20,187 WARN [IPC Server handler 3 on default port 44823 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-20T21:29:20,187 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 after 1ms 2024-11-20T21:29:22,057 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-20T21:29:22,057 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-20T21:29:22,058 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-20T21:29:22,058 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-20T21:29:22,555 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:41936 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37559:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41936 dst: /127.0.0.1:37559 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37559 remote=/127.0.0.1:41936]. Total timeout mills is 60000, 57599 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:22,555 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:48276 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48276 dst: /127.0.0.1:38023 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:22,555 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:45644 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:46339:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45644 dst: /127.0.0.1:46339 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:22,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741922_1106 (size=1794) 2024-11-20T21:29:22,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741922_1106 (size=1794) 2024-11-20T21:29:22,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741922_1106 (size=1794) 2024-11-20T21:29:23,852 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:29:24,188 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 after 4002ms 2024-11-20T21:29:24,216 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:24,216 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 took 4030ms 2024-11-20T21:29:24,218 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902; continuing. 2024-11-20T21:29:24,218 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 so closing down 2024-11-20T21:29:24,218 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:24,220 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732138159902.temp 2024-11-20T21:29:24,245 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000003-wal.1732138159902.temp 2024-11-20T21:29:24,246 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:24,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741928_1107 (size=800) 2024-11-20T21:29:24,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741928_1107 (size=800) 2024-11-20T21:29:24,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741928_1107 (size=800) 2024-11-20T21:29:24,252 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000003-wal.1732138159902.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:24,254 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000003-wal.1732138159902.temp to hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000008 2024-11-20T21:29:24,254 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T21:29:24,254 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902, journal: Splitting hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902, size=0 (0bytes) at 1732138160186Finishing writing output for hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 so closing down at 1732138164218 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000003-wal.1732138159902.temp at 1732138164245 (+27 ms)3 split writer threads finished at 1732138164246 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000003-wal.1732138159902.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732138164252 (+6 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000003-wal.1732138159902.temp to hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000008 at 1732138164254 (+2 ms)Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902, size=0, length=0, corrupted=false, cancelled=false at 1732138164254 2024-11-20T21:29:24,256 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138159902 2024-11-20T21:29:24,256 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000008 2024-11-20T21:29:24,257 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:24,258 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:24,271 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138164258, exclude list is [], retry=0 2024-11-20T21:29:24,274 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:24,274 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:24,274 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:24,302 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138164258 2024-11-20T21:29:24,302 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:24,302 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => db3cca17488e26661a37c81a08cc0d57, NAME => 'testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:24,303 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:24,303 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,303 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,304 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,305 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName a 2024-11-20T21:29:24,306 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,312 DEBUG [StoreFileOpener-db3cca17488e26661a37c81a08cc0d57-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T21:29:24,312 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_ 2024-11-20T21:29:24,316 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c 2024-11-20T21:29:24,319 DEBUG [StoreFileOpener-db3cca17488e26661a37c81a08cc0d57-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 65a972e7fb6e439bb32a74014c931b12_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T21:29:24,319 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_ 2024-11-20T21:29:24,322 DEBUG [StoreFileOpener-db3cca17488e26661a37c81a08cc0d57-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-20T21:29:24,322 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_ 2024-11-20T21:29:24,326 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/fc641b5d9dbd4fde81a88e0d19554a44 2024-11-20T21:29:24,326 WARN [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74c2af31 2024-11-20T21:29:24,326 WARN [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74c2af31 2024-11-20T21:29:24,326 WARN [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74c2af31 2024-11-20T21:29:24,326 WARN [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74c2af31 2024-11-20T21:29:24,326 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_] to archive 2024-11-20T21:29:24,327 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:29:24,329 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_ to hdfs://localhost:44823/hbase/archive/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_ 2024-11-20T21:29:24,330 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c to hdfs://localhost:44823/hbase/archive/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/35b1a8d0b07447fba4cb7664e8fb1f4c 2024-11-20T21:29:24,331 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_ to hdfs://localhost:44823/hbase/archive/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/65a972e7fb6e439bb32a74014c931b12_SeqId_4_ 2024-11-20T21:29:24,332 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_ to hdfs://localhost:44823/hbase/archive/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/a/7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_ 2024-11-20T21:29:24,332 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,332 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,333 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName b 2024-11-20T21:29:24,333 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,334 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,334 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,335 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db3cca17488e26661a37c81a08cc0d57 columnFamilyName c 2024-11-20T21:29:24,335 DEBUG [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,336 INFO [StoreOpener-db3cca17488e26661a37c81a08cc0d57-1 {}] regionserver.HStore(327): Store=db3cca17488e26661a37c81a08cc0d57/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,336 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,336 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,338 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,338 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000008 2024-11-20T21:29:24,366 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:24,368 DEBUG [Time-limited test {}] regionserver.HRegion(5836): db3cca17488e26661a37c81a08cc0d57 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "db3cca17488e26661a37c81a08cc0d57" family_name: "a" compaction_input: "35b1a8d0b07447fba4cb7664e8fb1f4c" compaction_input: "7a9a2b4c9286432cbc3abd7aeb8317d4_SeqId_4_" compaction_input: "65a972e7fb6e439bb32a74014c931b12_SeqId_4_" compaction_input: "15aaf1eb7b304ed0a929c10e0f0f4a90_SeqId_4_" compaction_output: "fc641b5d9dbd4fde81a88e0d19554a44" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-20T21:29:24,368 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-20T21:29:24,368 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000008 2024-11-20T21:29:24,369 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/0000000000000000008 2024-11-20T21:29:24,371 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,371 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,372 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:24,373 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db3cca17488e26661a37c81a08cc0d57 2024-11-20T21:29:24,376 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testCompactedBulkLoadedFiles/db3cca17488e26661a37c81a08cc0d57/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-20T21:29:24,377 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db3cca17488e26661a37c81a08cc0d57; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63385900, jitterRate=-0.055476486682891846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:24,377 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db3cca17488e26661a37c81a08cc0d57: Writing region info on filesystem at 1732138164303Initializing all the Stores at 1732138164304 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164304Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164304Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164304Cleaning up temporary data from old regions at 1732138164371 (+67 ms)Region opened successfully at 1732138164377 (+6 ms) 2024-11-20T21:29:24,380 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing db3cca17488e26661a37c81a08cc0d57, disabling compactions & flushes 2024-11-20T21:29:24,380 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:24,380 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:24,380 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. after waiting 0 ms 2024-11-20T21:29:24,380 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:24,381 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732138159813.db3cca17488e26661a37c81a08cc0d57. 2024-11-20T21:29:24,381 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for db3cca17488e26661a37c81a08cc0d57: Waiting for close lock at 1732138164380Disabling compacts and flushes for region at 1732138164380Disabling writes for close at 1732138164380Writing region close event to WAL at 1732138164381 (+1 ms)Closed at 1732138164381 2024-11-20T21:29:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741929_1108 (size=175) 2024-11-20T21:29:24,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741929_1108 (size=175) 2024-11-20T21:29:24,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741929_1108 (size=175) 2024-11-20T21:29:24,386 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:24,386 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732138164258) 2024-11-20T21:29:24,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=437 (was 426) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46269 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1654203666) connection to localhost/127.0.0.1:44823 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1388318444_22 at /127.0.0.1:45716 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1654203666) connection to localhost/127.0.0.1:46269 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1388318444_22 at /127.0.0.1:42034 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44823 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1388318444_22 at /127.0.0.1:48320 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=592 (was 635), ProcessCount=11 (was 11), AvailableMemoryMB=3100 (was 3179) 2024-11-20T21:29:24,405 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-20T21:29:24,423 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=592, ProcessCount=11, AvailableMemoryMB=3098 2024-11-20T21:29:24,423 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-20T21:29:24,446 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:24,448 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:29:24,449 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:29:24,451 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55694360, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/hregion-55694360, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:24,471 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55694360/hregion-55694360.1732138164452, exclude list is [], retry=0 2024-11-20T21:29:24,476 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:24,477 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:24,477 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:24,510 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55694360/hregion-55694360.1732138164452 2024-11-20T21:29:24,512 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:24,513 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bbb9c73756a5c7c9a255bea08efc5c4d, NAME => 'testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44823/hbase 2024-11-20T21:29:24,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741931_1110 (size=67) 2024-11-20T21:29:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741931_1110 (size=67) 2024-11-20T21:29:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741931_1110 (size=67) 2024-11-20T21:29:24,526 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:24,528 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,529 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName a 2024-11-20T21:29:24,529 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,530 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,530 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,531 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName b 2024-11-20T21:29:24,531 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,532 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,532 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,533 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName c 2024-11-20T21:29:24,533 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,534 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,534 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,534 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,535 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,536 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,536 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,536 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:24,537 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,539 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:29:24,540 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bbb9c73756a5c7c9a255bea08efc5c4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72665227, jitterRate=0.08279626071453094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:24,540 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bbb9c73756a5c7c9a255bea08efc5c4d: Writing region info on filesystem at 1732138164526Initializing all the Stores at 1732138164527 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164527Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164527Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164527Cleaning up temporary data from old regions at 1732138164536 (+9 ms)Region opened successfully at 1732138164540 (+4 ms) 2024-11-20T21:29:24,540 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bbb9c73756a5c7c9a255bea08efc5c4d, disabling compactions & flushes 2024-11-20T21:29:24,540 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,540 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,540 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. after waiting 0 ms 2024-11-20T21:29:24,540 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,541 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,541 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bbb9c73756a5c7c9a255bea08efc5c4d: Waiting for close lock at 1732138164540Disabling compacts and flushes for region at 1732138164540Disabling writes for close at 1732138164540Writing region close event to WAL at 1732138164540Closed at 1732138164540 2024-11-20T21:29:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741930_1109 (size=175) 2024-11-20T21:29:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741930_1109 (size=175) 2024-11-20T21:29:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741930_1109 (size=175) 2024-11-20T21:29:24,545 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:24,545 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55694360:(num 1732138164452) 2024-11-20T21:29:24,545 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:24,547 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:24,560 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547, exclude list is [], retry=0 2024-11-20T21:29:24,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:24,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:24,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:24,594 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547 2024-11-20T21:29:24,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:24,595 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bbb9c73756a5c7c9a255bea08efc5c4d, NAME => 'testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:24,595 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:24,595 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,595 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,596 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,597 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName a 2024-11-20T21:29:24,597 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,598 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,598 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,598 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName b 2024-11-20T21:29:24,599 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,599 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,599 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,600 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName c 2024-11-20T21:29:24,600 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,600 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,600 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,601 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,602 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,602 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,602 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,603 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:24,604 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,604 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bbb9c73756a5c7c9a255bea08efc5c4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69881423, jitterRate=0.041314348578453064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:24,604 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bbb9c73756a5c7c9a255bea08efc5c4d: Writing region info on filesystem at 1732138164595Initializing all the Stores at 1732138164596 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164596Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164596Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164596Cleaning up temporary data from old regions at 1732138164602 (+6 ms)Region opened successfully at 1732138164604 (+2 ms) 2024-11-20T21:29:24,613 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bbb9c73756a5c7c9a255bea08efc5c4d 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-20T21:29:24,629 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/a/47ad0f53f67e4fa793a2ae02cc07965e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732138164604/Put/seqid=0 2024-11-20T21:29:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741933_1112 (size=5958) 2024-11-20T21:29:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741933_1112 (size=5958) 2024-11-20T21:29:24,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741933_1112 (size=5958) 2024-11-20T21:29:24,636 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/a/47ad0f53f67e4fa793a2ae02cc07965e 2024-11-20T21:29:24,641 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/a/47ad0f53f67e4fa793a2ae02cc07965e as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/a/47ad0f53f67e4fa793a2ae02cc07965e 2024-11-20T21:29:24,645 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/a/47ad0f53f67e4fa793a2ae02cc07965e, entries=10, sequenceid=13, filesize=5.8 K 2024-11-20T21:29:24,646 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for bbb9c73756a5c7c9a255bea08efc5c4d in 33ms, sequenceid=13, compaction requested=false 2024-11-20T21:29:24,646 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bbb9c73756a5c7c9a255bea08efc5c4d: 2024-11-20T21:29:24,660 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bbb9c73756a5c7c9a255bea08efc5c4d, disabling compactions & flushes 2024-11-20T21:29:24,661 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,661 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,661 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. after waiting 0 ms 2024-11-20T21:29:24,661 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,661 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,661 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:24,661 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bbb9c73756a5c7c9a255bea08efc5c4d: Waiting for close lock at 1732138164660Disabling compacts and flushes for region at 1732138164660Disabling writes for close at 1732138164661 (+1 ms)Writing region close event to WAL at 1732138164661Closed at 1732138164661 2024-11-20T21:29:24,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741932_1111 (size=6243) 2024-11-20T21:29:24,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741932_1111 (size=6243) 2024-11-20T21:29:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741932_1111 (size=6243) 2024-11-20T21:29:24,679 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547, size=6.1 K (6243bytes) 2024-11-20T21:29:24,679 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547 2024-11-20T21:29:24,680 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547 after 0ms 2024-11-20T21:29:24,706 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:24,706 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547 took 27ms 2024-11-20T21:29:24,708 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547 so closing down 2024-11-20T21:29:24,708 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:24,709 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732138164547.temp 2024-11-20T21:29:24,735 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000003-wal.1732138164547.temp 2024-11-20T21:29:24,735 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:24,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741934_1113 (size=5575) 2024-11-20T21:29:24,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741934_1113 (size=5575) 2024-11-20T21:29:24,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741934_1113 (size=5575) 2024-11-20T21:29:24,743 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000003-wal.1732138164547.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:24,744 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000003-wal.1732138164547.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000035 2024-11-20T21:29:24,744 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-20T21:29:24,744 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547, journal: Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547, size=6.1 K (6243bytes) at 1732138164679Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547 so closing down at 1732138164708 (+29 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000003-wal.1732138164547.temp at 1732138164735 (+27 ms)3 split writer threads finished at 1732138164735Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000003-wal.1732138164547.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732138164743 (+8 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000003-wal.1732138164547.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000035 at 1732138164744 (+1 ms)Processed 32 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732138164744 2024-11-20T21:29:24,746 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164547 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138164547 2024-11-20T21:29:24,746 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000035 2024-11-20T21:29:24,747 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:24,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:24,761 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749, exclude list is [], retry=0 2024-11-20T21:29:24,763 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:24,764 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:24,764 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:24,800 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 2024-11-20T21:29:24,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:42083:42083),(127.0.0.1/127.0.0.1:39729:39729)] 2024-11-20T21:29:24,801 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bbb9c73756a5c7c9a255bea08efc5c4d, NAME => 'testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:29:24,801 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:24,801 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,801 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,803 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,804 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName a 2024-11-20T21:29:24,804 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,810 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/a/47ad0f53f67e4fa793a2ae02cc07965e 2024-11-20T21:29:24,810 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,810 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,811 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName b 2024-11-20T21:29:24,811 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,812 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,812 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,813 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName c 2024-11-20T21:29:24,813 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:24,813 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:24,814 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,814 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,816 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,816 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000035 2024-11-20T21:29:24,847 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:24,849 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000035 2024-11-20T21:29:24,849 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bbb9c73756a5c7c9a255bea08efc5c4d 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-20T21:29:24,869 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/b/7185580ad04c4a2db883b4b2af4c4642 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732138164646/Put/seqid=0 2024-11-20T21:29:24,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741936_1115 (size=5958) 2024-11-20T21:29:24,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741936_1115 (size=5958) 2024-11-20T21:29:24,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741936_1115 (size=5958) 2024-11-20T21:29:24,877 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/b/7185580ad04c4a2db883b4b2af4c4642 2024-11-20T21:29:24,897 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/c/a1d77e9bebf249d38a4d1c785b9d20e4 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732138164652/Put/seqid=0 2024-11-20T21:29:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741937_1116 (size=5958) 2024-11-20T21:29:24,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741937_1116 (size=5958) 2024-11-20T21:29:24,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741937_1116 (size=5958) 2024-11-20T21:29:24,909 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/c/a1d77e9bebf249d38a4d1c785b9d20e4 2024-11-20T21:29:24,916 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/b/7185580ad04c4a2db883b4b2af4c4642 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/b/7185580ad04c4a2db883b4b2af4c4642 2024-11-20T21:29:24,923 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/b/7185580ad04c4a2db883b4b2af4c4642, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T21:29:24,924 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/c/a1d77e9bebf249d38a4d1c785b9d20e4 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/c/a1d77e9bebf249d38a4d1c785b9d20e4 2024-11-20T21:29:24,930 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/c/a1d77e9bebf249d38a4d1c785b9d20e4, entries=10, sequenceid=35, filesize=5.8 K 2024-11-20T21:29:24,930 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for bbb9c73756a5c7c9a255bea08efc5c4d in 81ms, sequenceid=35, compaction requested=false; wal=null 2024-11-20T21:29:24,931 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000035 2024-11-20T21:29:24,932 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,932 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,933 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:24,934 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:24,937 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-20T21:29:24,938 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bbb9c73756a5c7c9a255bea08efc5c4d; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59808959, jitterRate=-0.10877706110477448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:24,938 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bbb9c73756a5c7c9a255bea08efc5c4d: Writing region info on filesystem at 1732138164801Initializing all the Stores at 1732138164802 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164802Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164803 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138164803Obtaining lock to block concurrent updates at 1732138164849 (+46 ms)Preparing flush snapshotting stores in bbb9c73756a5c7c9a255bea08efc5c4d at 1732138164849Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732138164849Flushing stores of testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. at 1732138164849Flushing bbb9c73756a5c7c9a255bea08efc5c4d/b: creating writer at 1732138164849Flushing bbb9c73756a5c7c9a255bea08efc5c4d/b: appending metadata at 1732138164868 (+19 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/b: closing flushed file at 1732138164869 (+1 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/c: creating writer at 1732138164882 (+13 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/c: appending metadata at 1732138164897 (+15 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/c: closing flushed file at 1732138164897Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@112a7049: reopening flushed file at 1732138164915 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bfe57d0: reopening flushed file at 1732138164923 (+8 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for bbb9c73756a5c7c9a255bea08efc5c4d in 81ms, sequenceid=35, compaction requested=false; wal=null at 1732138164930 (+7 ms)Cleaning up temporary data from old regions at 1732138164932 (+2 ms)Region opened successfully at 1732138164938 (+6 ms) 2024-11-20T21:29:25,020 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749, size=0 (0bytes) 2024-11-20T21:29:25,020 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 might be still open, length is 0 2024-11-20T21:29:25,020 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 2024-11-20T21:29:25,020 WARN [IPC Server handler 1 on default port 44823 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-20T21:29:25,021 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 after 1ms 2024-11-20T21:29:25,555 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:48360 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38023:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48360 dst: /127.0.0.1:38023 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38023 remote=/127.0.0.1:48360]. Total timeout mills is 60000, 59426 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:25,555 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:45788 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:46339:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45788 dst: /127.0.0.1:46339 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:25,556 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1057531477_22 at /127.0.0.1:42078 [Receiving block BP-422807794-172.17.0.3-1732138113638:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37559:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42078 dst: /127.0.0.1:37559 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:25,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741935_1117 (size=5567) 2024-11-20T21:29:25,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741935_1117 (size=5567) 2024-11-20T21:29:27,445 INFO [master/0b16ab3f13b7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T21:29:27,445 INFO [master/0b16ab3f13b7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T21:29:29,021 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 after 4001ms 2024-11-20T21:29:29,049 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:29,050 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 took 4030ms 2024-11-20T21:29:29,052 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749; continuing. 2024-11-20T21:29:29,052 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 so closing down 2024-11-20T21:29:29,052 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T21:29:29,053 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732138164749.temp 2024-11-20T21:29:29,080 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000037-wal.1732138164749.temp 2024-11-20T21:29:29,080 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T21:29:29,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741938_1118 (size=5575) 2024-11-20T21:29:29,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741938_1118 (size=5575) 2024-11-20T21:29:29,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741938_1118 (size=5575) 2024-11-20T21:29:29,100 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000037-wal.1732138164749.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-20T21:29:29,101 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000037-wal.1732138164749.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000066 2024-11-20T21:29:29,101 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749, size=0, length=0, corrupted=false, cancelled=false 2024-11-20T21:29:29,101 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749, journal: Splitting hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749, size=0 (0bytes) at 1732138165020Finishing writing output for hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 so closing down at 1732138169052 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000037-wal.1732138164749.temp at 1732138169080 (+28 ms)3 split writer threads finished at 1732138169080Closed recovered edits writer path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000037-wal.1732138164749.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732138169100 (+20 ms)Rename recovered edits hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000037-wal.1732138164749.temp to hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000066 at 1732138169101 (+1 ms)Processed 30 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749, size=0, length=0, corrupted=false, cancelled=false at 1732138169101 2024-11-20T21:29:29,103 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 to hdfs://localhost:44823/hbase/oldWALs/wal.1732138164749 2024-11-20T21:29:29,104 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000066 2024-11-20T21:29:29,104 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-20T21:29:29,105 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44823/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445, archiveDir=hdfs://localhost:44823/hbase/oldWALs, maxLogs=32 2024-11-20T21:29:29,118 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138169106, exclude list is [], retry=0 2024-11-20T21:29:29,120 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38023,DS-1079942e-6119-44ed-afe9-b5dcceee23cc,DISK] 2024-11-20T21:29:29,120 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37559,DS-226e5c0c-b53a-46f8-87d0-7e3993a4f63c,DISK] 2024-11-20T21:29:29,121 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46339,DS-76880aaf-5eec-442b-b195-5d588f61907b,DISK] 2024-11-20T21:29:29,146 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138169106 2024-11-20T21:29:29,146 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34363:34363),(127.0.0.1/127.0.0.1:39729:39729),(127.0.0.1/127.0.0.1:42083:42083)] 2024-11-20T21:29:29,147 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:29:29,148 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,149 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName a 2024-11-20T21:29:29,149 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:29,154 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/a/47ad0f53f67e4fa793a2ae02cc07965e 2024-11-20T21:29:29,154 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:29,154 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,155 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName b 2024-11-20T21:29:29,155 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:29,159 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/b/7185580ad04c4a2db883b4b2af4c4642 2024-11-20T21:29:29,159 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:29,159 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,160 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbb9c73756a5c7c9a255bea08efc5c4d columnFamilyName c 2024-11-20T21:29:29,160 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:29:29,164 DEBUG [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/c/a1d77e9bebf249d38a4d1c785b9d20e4 2024-11-20T21:29:29,164 INFO [StoreOpener-bbb9c73756a5c7c9a255bea08efc5c4d-1 {}] regionserver.HStore(327): Store=bbb9c73756a5c7c9a255bea08efc5c4d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:29:29,165 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,165 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,167 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,167 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000066 2024-11-20T21:29:29,193 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-20T21:29:29,198 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000066 2024-11-20T21:29:29,198 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bbb9c73756a5c7c9a255bea08efc5c4d 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-20T21:29:29,212 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/a/22fc85a40c19475c9da7ee4ced167145 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732138164947/Put/seqid=0 2024-11-20T21:29:29,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741940_1120 (size=5958) 2024-11-20T21:29:29,219 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/a/22fc85a40c19475c9da7ee4ced167145 2024-11-20T21:29:29,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741940_1120 (size=5958) 2024-11-20T21:29:29,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741940_1120 (size=5958) 2024-11-20T21:29:29,237 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/b/a1f95682083a4830a740915a6183e3cc is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732138164960/Put/seqid=0 2024-11-20T21:29:29,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741941_1121 (size=5958) 2024-11-20T21:29:29,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741941_1121 (size=5958) 2024-11-20T21:29:29,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741941_1121 (size=5958) 2024-11-20T21:29:29,243 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/b/a1f95682083a4830a740915a6183e3cc 2024-11-20T21:29:29,262 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/c/360eb7ca32564f3ca4a2e855c8053647 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732138164972/Put/seqid=0 2024-11-20T21:29:29,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741942_1122 (size=5958) 2024-11-20T21:29:29,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741942_1122 (size=5958) 2024-11-20T21:29:29,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741942_1122 (size=5958) 2024-11-20T21:29:29,268 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/c/360eb7ca32564f3ca4a2e855c8053647 2024-11-20T21:29:29,274 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/a/22fc85a40c19475c9da7ee4ced167145 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/a/22fc85a40c19475c9da7ee4ced167145 2024-11-20T21:29:29,278 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/a/22fc85a40c19475c9da7ee4ced167145, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T21:29:29,279 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/b/a1f95682083a4830a740915a6183e3cc as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/b/a1f95682083a4830a740915a6183e3cc 2024-11-20T21:29:29,284 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/b/a1f95682083a4830a740915a6183e3cc, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T21:29:29,284 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/.tmp/c/360eb7ca32564f3ca4a2e855c8053647 as hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/c/360eb7ca32564f3ca4a2e855c8053647 2024-11-20T21:29:29,288 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/c/360eb7ca32564f3ca4a2e855c8053647, entries=10, sequenceid=66, filesize=5.8 K 2024-11-20T21:29:29,289 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for bbb9c73756a5c7c9a255bea08efc5c4d in 91ms, sequenceid=66, compaction requested=false; wal=null 2024-11-20T21:29:29,289 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/0000000000000000066 2024-11-20T21:29:29,290 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,290 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,291 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-20T21:29:29,292 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bbb9c73756a5c7c9a255bea08efc5c4d 2024-11-20T21:29:29,293 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/hbase/data/default/testReplayEditsWrittenViaHRegion/bbb9c73756a5c7c9a255bea08efc5c4d/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-20T21:29:29,294 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bbb9c73756a5c7c9a255bea08efc5c4d; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70449611, jitterRate=0.049781009554862976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-20T21:29:29,294 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bbb9c73756a5c7c9a255bea08efc5c4d: Writing region info on filesystem at 1732138169147Initializing all the Stores at 1732138169148 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138169148Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138169148Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732138169148Obtaining lock to block concurrent updates at 1732138169198 (+50 ms)Preparing flush snapshotting stores in bbb9c73756a5c7c9a255bea08efc5c4d at 1732138169198Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732138169198Flushing stores of testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. at 1732138169198Flushing bbb9c73756a5c7c9a255bea08efc5c4d/a: creating writer at 1732138169198Flushing bbb9c73756a5c7c9a255bea08efc5c4d/a: appending metadata at 1732138169211 (+13 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/a: closing flushed file at 1732138169211Flushing bbb9c73756a5c7c9a255bea08efc5c4d/b: creating writer at 1732138169223 (+12 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/b: appending metadata at 1732138169236 (+13 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/b: closing flushed file at 1732138169236Flushing bbb9c73756a5c7c9a255bea08efc5c4d/c: creating writer at 1732138169248 (+12 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/c: appending metadata at 1732138169261 (+13 ms)Flushing bbb9c73756a5c7c9a255bea08efc5c4d/c: closing flushed file at 1732138169261Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@675d26d3: reopening flushed file at 1732138169273 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fd8e9af: reopening flushed file at 1732138169278 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28b7ef1a: reopening flushed file at 1732138169284 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for bbb9c73756a5c7c9a255bea08efc5c4d in 91ms, sequenceid=66, compaction requested=false; wal=null at 1732138169289 (+5 ms)Cleaning up temporary data from old regions at 1732138169290 (+1 ms)Region opened successfully at 1732138169294 (+4 ms) 2024-11-20T21:29:29,306 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bbb9c73756a5c7c9a255bea08efc5c4d, disabling compactions & flushes 2024-11-20T21:29:29,306 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:29,306 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:29,307 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. after waiting 0 ms 2024-11-20T21:29:29,307 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:29,308 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732138164447.bbb9c73756a5c7c9a255bea08efc5c4d. 2024-11-20T21:29:29,308 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bbb9c73756a5c7c9a255bea08efc5c4d: Waiting for close lock at 1732138169306Disabling compacts and flushes for region at 1732138169306Disabling writes for close at 1732138169307 (+1 ms)Writing region close event to WAL at 1732138169308 (+1 ms)Closed at 1732138169308 2024-11-20T21:29:29,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741939_1119 (size=175) 2024-11-20T21:29:29,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741939_1119 (size=175) 2024-11-20T21:29:29,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741939_1119 (size=175) 2024-11-20T21:29:29,312 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-20T21:29:29,312 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732138169106) 2024-11-20T21:29:29,328 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=441 (was 437) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1865730751_22 at /127.0.0.1:57680 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1865730751_22 at /127.0.0.1:37460 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1865730751_22 at /127.0.0.1:52012 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1654203666) connection to localhost/127.0.0.1:44823 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44823 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=561 (was 592), ProcessCount=11 (was 11), AvailableMemoryMB=2416 (was 3098) 2024-11-20T21:29:29,329 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-20T21:29:29,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T21:29:29,329 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:29:29,329 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:29:29,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:29,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:29,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:29:29,330 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T21:29:29,330 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1651798009, stopped=false 2024-11-20T21:29:29,330 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b16ab3f13b7,39535,1732138121558 2024-11-20T21:29:29,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:29:29,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:29:29,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:29:29,333 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:29:29,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:29:29,334 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:29:29,334 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:29:29,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:29,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:29:29,334 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b16ab3f13b7,45827,1732138122912' ***** 2024-11-20T21:29:29,334 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T21:29:29,334 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b16ab3f13b7,40171,1732138123009' ***** 2024-11-20T21:29:29,334 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T21:29:29,335 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:29:29,335 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:29:29,335 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:29:29,335 INFO [RS:2;0b16ab3f13b7:40171 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:29:29,335 INFO [RS:2;0b16ab3f13b7:40171 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:29:29,335 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(959): stopping server 0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:29,336 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:29:29,336 INFO [RS:2;0b16ab3f13b7:40171 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0b16ab3f13b7:40171. 2024-11-20T21:29:29,336 DEBUG [RS:2;0b16ab3f13b7:40171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:29:29,336 DEBUG [RS:2;0b16ab3f13b7:40171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:29,336 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:29:29,336 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:29:29,336 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:29:29,336 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T21:29:29,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:29:29,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:29:29,337 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T21:29:29,337 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:29:29,337 DEBUG [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(3091): Received CLOSE for 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(959): stopping server 0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0b16ab3f13b7:45827. 2024-11-20T21:29:29,337 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06f91812474a7664c203be2e75d69d0e, disabling compactions & flushes 2024-11-20T21:29:29,337 DEBUG [RS:1;0b16ab3f13b7:45827 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:29:29,337 DEBUG [RS:1;0b16ab3f13b7:45827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:29,337 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:29,337 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:29,337 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T21:29:29,338 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. after waiting 0 ms 2024-11-20T21:29:29,338 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1325): Online Regions={06f91812474a7664c203be2e75d69d0e=testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e.} 2024-11-20T21:29:29,338 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:29,338 DEBUG [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1351): Waiting on 06f91812474a7664c203be2e75d69d0e 2024-11-20T21:29:29,338 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:29:29,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:29:29,338 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:29:29,338 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:29:29,338 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:29:29,338 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:29:29,338 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:29:29,338 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-20T21:29:29,348 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/default/testReplayEditsAfterRegionMovedWithMultiCF/06f91812474a7664c203be2e75d69d0e/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-20T21:29:29,349 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:29,349 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06f91812474a7664c203be2e75d69d0e: Waiting for close lock at 1732138169337Running coprocessor pre-close hooks at 1732138169337Disabling compacts and flushes for region at 1732138169337Disabling writes for close at 1732138169338 (+1 ms)Writing region close event to WAL at 1732138169339 (+1 ms)Running coprocessor post-close hooks at 1732138169349 (+10 ms)Closed at 1732138169349 2024-11-20T21:29:29,349 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e. 2024-11-20T21:29:29,356 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/info/ca521d1c0b6244639f6fd544b583a427 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732138142851.06f91812474a7664c203be2e75d69d0e./info:regioninfo/1732138147728/Put/seqid=0 2024-11-20T21:29:29,359 INFO [regionserver/0b16ab3f13b7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:29:29,359 INFO [regionserver/0b16ab3f13b7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:29:29,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741943_1123 (size=6778) 2024-11-20T21:29:29,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741943_1123 (size=6778) 2024-11-20T21:29:29,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741943_1123 (size=6778) 2024-11-20T21:29:29,368 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/info/ca521d1c0b6244639f6fd544b583a427 2024-11-20T21:29:29,373 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/.tmp/info/ca521d1c0b6244639f6fd544b583a427 as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/info/ca521d1c0b6244639f6fd544b583a427 2024-11-20T21:29:29,378 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/info/ca521d1c0b6244639f6fd544b583a427, entries=8, sequenceid=23, filesize=6.6 K 2024-11-20T21:29:29,380 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=23, compaction requested=false 2024-11-20T21:29:29,380 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T21:29:29,387 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-20T21:29:29,388 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:29:29,388 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:29:29,388 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732138169338Running coprocessor pre-close hooks at 1732138169338Disabling compacts and flushes for region at 1732138169338Disabling writes for close at 1732138169338Obtaining lock to block concurrent updates at 1732138169338Preparing flush snapshotting stores in 1588230740 at 1732138169338Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732138169339 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732138169340 (+1 ms)Flushing 1588230740/info: creating writer at 1732138169340Flushing 1588230740/info: appending metadata at 1732138169355 (+15 ms)Flushing 1588230740/info: closing flushed file at 1732138169355Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@736932ec: reopening flushed file at 1732138169373 (+18 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=23, compaction requested=false at 1732138169380 (+7 ms)Writing region close event to WAL at 1732138169384 (+4 ms)Running coprocessor post-close hooks at 1732138169388 (+4 ms)Closed at 1732138169388 2024-11-20T21:29:29,388 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T21:29:29,537 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(976): stopping server 0b16ab3f13b7,40171,1732138123009; all regions closed. 2024-11-20T21:29:29,538 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(976): stopping server 0b16ab3f13b7,45827,1732138122912; all regions closed. 2024-11-20T21:29:29,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741893_1071 (size=2625) 2024-11-20T21:29:29,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741893_1071 (size=2625) 2024-11-20T21:29:29,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741893_1071 (size=2625) 2024-11-20T21:29:29,544 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/WALs/0b16ab3f13b7,45827,1732138122912/0b16ab3f13b7%2C45827%2C1732138122912.1732138125950 not finished, retry = 0 2024-11-20T21:29:29,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741833_1009 (size=3621) 2024-11-20T21:29:29,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741833_1009 (size=3621) 2024-11-20T21:29:29,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741833_1009 (size=3621) 2024-11-20T21:29:29,553 DEBUG [RS:2;0b16ab3f13b7:40171 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs 2024-11-20T21:29:29,553 INFO [RS:2;0b16ab3f13b7:40171 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b16ab3f13b7%2C40171%2C1732138123009.meta:.meta(num 1732138146717) 2024-11-20T21:29:29,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741835_1011 (size=175) 2024-11-20T21:29:29,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741835_1011 (size=175) 2024-11-20T21:29:29,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741835_1011 (size=175) 2024-11-20T21:29:29,563 DEBUG [RS:2;0b16ab3f13b7:40171 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs 2024-11-20T21:29:29,563 INFO [RS:2;0b16ab3f13b7:40171 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b16ab3f13b7%2C40171%2C1732138123009:(num 1732138125950) 2024-11-20T21:29:29,563 DEBUG [RS:2;0b16ab3f13b7:40171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:29,563 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:29:29,563 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:29:29,564 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.ChoreService(370): Chore service for: regionserver/0b16ab3f13b7:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T21:29:29,564 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:29:29,564 INFO [regionserver/0b16ab3f13b7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:29:29,564 INFO [RS:2;0b16ab3f13b7:40171 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40171 2024-11-20T21:29:29,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:29:29,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b16ab3f13b7,40171,1732138123009 2024-11-20T21:29:29,566 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:29:29,567 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b16ab3f13b7,40171,1732138123009] 2024-11-20T21:29:29,569 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b16ab3f13b7,40171,1732138123009 already deleted, retry=false 2024-11-20T21:29:29,569 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b16ab3f13b7,40171,1732138123009 expired; onlineServers=1 2024-11-20T21:29:29,646 DEBUG [RS:1;0b16ab3f13b7:45827 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/oldWALs 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b16ab3f13b7%2C45827%2C1732138122912:(num 1732138125950) 2024-11-20T21:29:29,647 DEBUG [RS:1;0b16ab3f13b7:45827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.ChoreService(370): Chore service for: regionserver/0b16ab3f13b7:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:29:29,647 INFO [regionserver/0b16ab3f13b7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:29:29,647 INFO [RS:1;0b16ab3f13b7:45827 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45827 2024-11-20T21:29:29,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b16ab3f13b7,45827,1732138122912 2024-11-20T21:29:29,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:29:29,650 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:29:29,650 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b16ab3f13b7,45827,1732138122912] 2024-11-20T21:29:29,653 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b16ab3f13b7,45827,1732138122912 already deleted, retry=false 2024-11-20T21:29:29,653 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b16ab3f13b7,45827,1732138122912 expired; onlineServers=0 2024-11-20T21:29:29,653 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b16ab3f13b7,39535,1732138121558' ***** 2024-11-20T21:29:29,653 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T21:29:29,653 INFO [M:0;0b16ab3f13b7:39535 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:29:29,653 INFO [M:0;0b16ab3f13b7:39535 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:29:29,653 DEBUG [M:0;0b16ab3f13b7:39535 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T21:29:29,653 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T21:29:29,653 DEBUG [M:0;0b16ab3f13b7:39535 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T21:29:29,653 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.small.0-1732138125253 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.small.0-1732138125253,5,FailOnTimeoutGroup] 2024-11-20T21:29:29,653 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.large.0-1732138125251 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.large.0-1732138125251,5,FailOnTimeoutGroup] 2024-11-20T21:29:29,653 INFO [M:0;0b16ab3f13b7:39535 {}] hbase.ChoreService(370): Chore service for: master/0b16ab3f13b7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T21:29:29,654 INFO [M:0;0b16ab3f13b7:39535 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:29:29,654 DEBUG [M:0;0b16ab3f13b7:39535 {}] master.HMaster(1795): Stopping service threads 2024-11-20T21:29:29,654 INFO [M:0;0b16ab3f13b7:39535 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T21:29:29,654 INFO [M:0;0b16ab3f13b7:39535 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:29:29,654 INFO [M:0;0b16ab3f13b7:39535 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T21:29:29,655 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T21:29:29,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T21:29:29,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:29:29,656 DEBUG [M:0;0b16ab3f13b7:39535 {}] zookeeper.ZKUtil(347): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T21:29:29,656 WARN [M:0;0b16ab3f13b7:39535 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T21:29:29,657 INFO [M:0;0b16ab3f13b7:39535 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/.lastflushedseqids 2024-11-20T21:29:29,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:29,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40171-0x1005f1840a00003, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:29,668 INFO [RS:2;0b16ab3f13b7:40171 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:29:29,668 INFO [RS:2;0b16ab3f13b7:40171 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b16ab3f13b7,40171,1732138123009; zookeeper connection closed. 2024-11-20T21:29:29,669 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@64a91401 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@64a91401 2024-11-20T21:29:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741944_1124 (size=119) 2024-11-20T21:29:29,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741944_1124 (size=119) 2024-11-20T21:29:29,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741944_1124 (size=119) 2024-11-20T21:29:29,671 INFO [M:0;0b16ab3f13b7:39535 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T21:29:29,671 INFO [M:0;0b16ab3f13b7:39535 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T21:29:29,672 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:29:29,672 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:29:29,672 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:29:29,672 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:29:29,672 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:29:29,672 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-11-20T21:29:29,688 DEBUG [M:0;0b16ab3f13b7:39535 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/abb50ccd2b564272ac56e11961bcf78a is 82, key is hbase:meta,,1/info:regioninfo/1732138147005/Put/seqid=0 2024-11-20T21:29:29,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741945_1125 (size=6063) 2024-11-20T21:29:29,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741945_1125 (size=6063) 2024-11-20T21:29:29,696 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/abb50ccd2b564272ac56e11961bcf78a 2024-11-20T21:29:29,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741945_1125 (size=6063) 2024-11-20T21:29:29,718 DEBUG [M:0;0b16ab3f13b7:39535 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8422c327fffd486881ae2c078661354f is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732138143699/Put/seqid=0 2024-11-20T21:29:29,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741946_1126 (size=7906) 2024-11-20T21:29:29,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741946_1126 (size=7906) 2024-11-20T21:29:29,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741946_1126 (size=7906) 2024-11-20T21:29:29,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:29,752 INFO [RS:1;0b16ab3f13b7:45827 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:29:29,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45827-0x1005f1840a00002, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:29,752 INFO [RS:1;0b16ab3f13b7:45827 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b16ab3f13b7,45827,1732138122912; zookeeper connection closed. 2024-11-20T21:29:29,752 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@463c2c4d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@463c2c4d 2024-11-20T21:29:29,753 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T21:29:30,126 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8422c327fffd486881ae2c078661354f 2024-11-20T21:29:30,131 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8422c327fffd486881ae2c078661354f 2024-11-20T21:29:30,147 DEBUG [M:0;0b16ab3f13b7:39535 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/18d974a1001741be8e6bcac335f306bc is 69, key is 0b16ab3f13b7,40171,1732138123009/rs:state/1732138125495/Put/seqid=0 2024-11-20T21:29:30,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741947_1127 (size=5440) 2024-11-20T21:29:30,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741947_1127 (size=5440) 2024-11-20T21:29:30,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741947_1127 (size=5440) 2024-11-20T21:29:30,153 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/18d974a1001741be8e6bcac335f306bc 2024-11-20T21:29:30,157 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 18d974a1001741be8e6bcac335f306bc 2024-11-20T21:29:30,158 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/abb50ccd2b564272ac56e11961bcf78a as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/abb50ccd2b564272ac56e11961bcf78a 2024-11-20T21:29:30,162 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/abb50ccd2b564272ac56e11961bcf78a, entries=14, sequenceid=207, filesize=5.9 K 2024-11-20T21:29:30,163 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8422c327fffd486881ae2c078661354f as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8422c327fffd486881ae2c078661354f 2024-11-20T21:29:30,167 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8422c327fffd486881ae2c078661354f 2024-11-20T21:29:30,168 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8422c327fffd486881ae2c078661354f, entries=21, sequenceid=207, filesize=7.7 K 2024-11-20T21:29:30,168 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/18d974a1001741be8e6bcac335f306bc as hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/18d974a1001741be8e6bcac335f306bc 2024-11-20T21:29:30,172 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 18d974a1001741be8e6bcac335f306bc 2024-11-20T21:29:30,172 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44823/user/jenkins/test-data/00df85db-ce0a-f3ef-2171-ddfb3a50781c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/18d974a1001741be8e6bcac335f306bc, entries=3, sequenceid=207, filesize=5.3 K 2024-11-20T21:29:30,173 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 501ms, sequenceid=207, compaction requested=false 2024-11-20T21:29:30,174 INFO [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:29:30,175 DEBUG [M:0;0b16ab3f13b7:39535 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732138169672Disabling compacts and flushes for region at 1732138169672Disabling writes for close at 1732138169672Obtaining lock to block concurrent updates at 1732138169672Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732138169672Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1732138169672Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732138169673 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732138169673Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732138169687 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732138169687Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732138169701 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732138169717 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732138169717Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732138170131 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732138170145 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732138170145Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aaecb24: reopening flushed file at 1732138170158 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@391de673: reopening flushed file at 1732138170163 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bdfcf95: reopening flushed file at 1732138170168 (+5 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 501ms, sequenceid=207, compaction requested=false at 1732138170173 (+5 ms)Writing region close event to WAL at 1732138170174 (+1 ms)Closed at 1732138170174 2024-11-20T21:29:30,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46339 is added to blk_1073741830_1006 (size=102150) 2024-11-20T21:29:30,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37559 is added to blk_1073741830_1006 (size=102150) 2024-11-20T21:29:30,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38023 is added to blk_1073741830_1006 (size=102150) 2024-11-20T21:29:30,178 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:29:30,178 INFO [M:0;0b16ab3f13b7:39535 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T21:29:30,178 INFO [M:0;0b16ab3f13b7:39535 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39535 2024-11-20T21:29:30,178 INFO [M:0;0b16ab3f13b7:39535 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:29:30,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:30,280 INFO [M:0;0b16ab3f13b7:39535 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:29:30,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x1005f1840a00000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:29:30,287 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138158596 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138158596 (inode 16688) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732138149084/wal.1732138158596 (inode 16688) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T21:29:30,290 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148754 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148754 (inode 16665) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732138148278/wal.1732138148754 (inode 16665) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T21:29:30,291 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732138138206/wal.1732138138350 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:30,293 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138159614 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138159614 (inode 16714) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732138158885/wal.1732138159614 (inode 16714) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T21:29:30,293 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732138159812/wal.1732138159902 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:30,293 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732138164445/wal.1732138164749 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:30,294 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732138129423/wal.1732138129576 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:29:30,296 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal.1732138129118 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal.1732138129118 (inode 16485) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732138128153/wal.1732138129118 (inode 16485) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T21:29:30,298 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal.1732138127786 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal.1732138127786 (inode 16462) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732138127049/wal.1732138127786 (inode 16462) Holder DFSClient_NONMAPREDUCE_1057531477_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-20T21:29:30,303 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10e5d78d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:29:30,306 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@148e0151{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:29:30,306 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:29:30,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e14f56{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:29:30,306 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b0205f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,STOPPED} 2024-11-20T21:29:30,308 WARN [BP-422807794-172.17.0.3-1732138113638 heartbeating to localhost/127.0.0.1:44823 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T21:29:30,309 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T21:29:30,309 WARN [BP-422807794-172.17.0.3-1732138113638 heartbeating to localhost/127.0.0.1:44823 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-422807794-172.17.0.3-1732138113638 (Datanode Uuid 147809d0-3968-4205-9fd0-23adeea48b00) service to localhost/127.0.0.1:44823 2024-11-20T21:29:30,309 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T21:29:30,310 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data5/current/BP-422807794-172.17.0.3-1732138113638 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:29:30,310 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data6/current/BP-422807794-172.17.0.3-1732138113638 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:29:30,310 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T21:29:30,312 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5bccc2bb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:29:30,313 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c0acdf3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:29:30,313 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:29:30,313 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2bea299b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:29:30,313 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b30c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,STOPPED} 2024-11-20T21:29:30,314 WARN [BP-422807794-172.17.0.3-1732138113638 heartbeating to localhost/127.0.0.1:44823 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T21:29:30,314 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T21:29:30,314 WARN [BP-422807794-172.17.0.3-1732138113638 heartbeating to localhost/127.0.0.1:44823 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-422807794-172.17.0.3-1732138113638 (Datanode Uuid fb3d072d-f033-45f3-8fe2-0eabc36e7a26) service to localhost/127.0.0.1:44823 2024-11-20T21:29:30,314 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T21:29:30,315 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data3/current/BP-422807794-172.17.0.3-1732138113638 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:29:30,315 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data4/current/BP-422807794-172.17.0.3-1732138113638 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:29:30,315 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T21:29:30,318 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c6d8c3d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:29:30,318 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@56e06eb3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:29:30,318 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:29:30,318 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ca6fdcd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:29:30,318 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@615a9ad9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,STOPPED} 2024-11-20T21:29:30,320 WARN [BP-422807794-172.17.0.3-1732138113638 heartbeating to localhost/127.0.0.1:44823 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T21:29:30,320 WARN [BP-422807794-172.17.0.3-1732138113638 heartbeating to localhost/127.0.0.1:44823 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-422807794-172.17.0.3-1732138113638 (Datanode Uuid db13ba50-1d12-4269-95db-acb57a9aa803) service to localhost/127.0.0.1:44823 2024-11-20T21:29:30,321 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data1/current/BP-422807794-172.17.0.3-1732138113638 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:29:30,321 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/cluster_681f8bf8-dccf-f819-9864-a57250a98e59/data/data2/current/BP-422807794-172.17.0.3-1732138113638 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:29:30,321 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T21:29:30,324 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T21:29:30,324 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T21:29:30,328 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5cbdf8c4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T21:29:30,328 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@655f9c10{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:29:30,328 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:29:30,328 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b2e6df8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:29:30,329 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a22ae35{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/93c5c644-8694-c41b-d67e-a92b51d963ad/hadoop.log.dir/,STOPPED} 2024-11-20T21:29:30,341 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T21:29:30,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down